test_minio_backend.py
python
sha256:eead4146ec6c9905a097a89f8dbffa3d5c5e8ef9c1acd0e8a5b2a93b0084d273
Mpack content-addressability
Human
13 hours ago
| 1 | """MinIO — BlobBackend as the only storage backend. |
| 2 | |
| 3 | Tier 1 — Unit (no network) |
| 4 | - get_backend() raises RuntimeError when no bucket is configured |
| 5 | - get_backend() returns BlobBackend when blob_storage_bucket is set |
| 6 | - get_backend() returns BlobBackend when aws_s3_asset_bucket is set |
| 7 | - LocalBackend is not importable (deleted) |
| 8 | |
| 9 | Tier 2 — Integration (MinIO required) |
| 10 | - BlobBackend.put/get/exists/delete round-trip against a live MinIO container |
| 11 | - put is idempotent (second write is safe) |
| 12 | - get returns None for missing object |
| 13 | - exists returns False for missing object |
| 14 | |
| 15 | Tier 3 — Mpack content-addressability (MinIO required) |
| 16 | - put_mpack/get_mpack round-trip: sha256(get_mpack(key)) == key |
| 17 | - presign_mpack_put PUT/get_mpack round-trip: sha256(get_mpack(key)) == key |
| 18 | |
| 19 | MinIO assumed at http://localhost:9000 with credentials minioadmin/minioadmin. |
| 20 | Tests in TestBlobBackendMinIO are skipped automatically when MinIO is unreachable. |
| 21 | """ |
| 22 | from __future__ import annotations |
| 23 | |
| 24 | import hashlib |
| 25 | import importlib |
| 26 | import secrets |
| 27 | import urllib.request |
| 28 | from unittest.mock import patch |
| 29 | |
| 30 | import pytest |
| 31 | |
| 32 | from muse.core.types import long_id |
| 33 | |
| 34 | |
| 35 | # ─── helpers ────────────────────────────────────────────────────────────────── |
| 36 | |
| 37 | MINIO_ENDPOINT = "http://localhost:9000" |
| 38 | MINIO_BUCKET = "muse-objects" |
| 39 | MINIO_ACCESS_KEY = "minioadmin" |
| 40 | MINIO_SECRET_KEY = "minioadmin" |
| 41 | |
| 42 | |
| 43 | def _oid() -> str: |
| 44 | return long_id(secrets.token_hex(32)) |
| 45 | |
| 46 | |
| 47 | def _minio_backend() -> None: |
| 48 | from musehub.storage.backends import BlobBackend |
| 49 | return BlobBackend( |
| 50 | bucket=MINIO_BUCKET, |
| 51 | endpoint_url=MINIO_ENDPOINT, |
| 52 | access_key_id=MINIO_ACCESS_KEY, |
| 53 | secret_access_key=MINIO_SECRET_KEY, |
| 54 | region="us-east-1", |
| 55 | ) |
| 56 | |
| 57 | |
| 58 | def _minio_reachable() -> bool: |
| 59 | try: |
| 60 | import urllib.request |
| 61 | urllib.request.urlopen(f"{MINIO_ENDPOINT}/minio/health/live", timeout=2) |
| 62 | return True |
| 63 | except Exception: |
| 64 | return False |
| 65 | |
| 66 | |
| 67 | requires_minio = pytest.mark.skipif( |
| 68 | not _minio_reachable(), |
| 69 | reason="MinIO not reachable at localhost:9000 — start docker compose first", |
| 70 | ) |
| 71 | |
| 72 | |
| 73 | # ═══════════════════════════════════════════════════════════════════════════════ |
| 74 | # Tier 1 — Unit: get_backend() contract after LocalBackend is deleted |
| 75 | # ═══════════════════════════════════════════════════════════════════════════════ |
| 76 | |
| 77 | |
| 78 | class TestBlobBackendNaming: |
| 79 | def test_blob_backend_importable(self) -> None: |
| 80 | from musehub.storage.backends import BlobBackend # noqa: F401 |
| 81 | |
| 82 | def test_blob_backend_exported_from_storage(self) -> None: |
| 83 | import musehub.storage as mod |
| 84 | assert hasattr(mod, "BlobBackend") |
| 85 | |
| 86 | def test_s3_backend_name_deleted(self) -> None: |
| 87 | """S3Backend must not exist — the name leaks the transport protocol.""" |
| 88 | import musehub.storage.backends as mod |
| 89 | assert not hasattr(mod, "S3Backend"), ( |
| 90 | "S3Backend still exists — rename it to BlobBackend" |
| 91 | ) |
| 92 | |
| 93 | def test_get_backend_returns_blob_backend(self) -> None: |
| 94 | from musehub.storage.backends import BlobBackend, get_backend |
| 95 | from unittest.mock import patch |
| 96 | with patch("musehub.storage.backends.settings") as mock_settings: |
| 97 | mock_settings.blob_storage_bucket = "muse-objects" |
| 98 | mock_settings.blob_storage_endpoint = MINIO_ENDPOINT |
| 99 | mock_settings.blob_storage_access_key_id = MINIO_ACCESS_KEY |
| 100 | mock_settings.blob_storage_secret_access_key = MINIO_SECRET_KEY |
| 101 | mock_settings.blob_storage_region = "us-east-1" |
| 102 | result = get_backend() |
| 103 | assert isinstance(result, BlobBackend) |
| 104 | |
| 105 | |
| 106 | class TestGetBackendNoFallback: |
| 107 | def test_raises_when_no_bucket_configured(self) -> None: |
| 108 | """get_backend() must raise RuntimeError when neither blob_storage_bucket nor |
| 109 | aws_s3_asset_bucket is set. No silent LocalBackend fallback.""" |
| 110 | from musehub.storage.backends import _get_backend_impl |
| 111 | with patch("musehub.storage.backends.settings") as mock_settings: |
| 112 | mock_settings.blob_storage_bucket = None |
| 113 | mock_settings.aws_s3_asset_bucket = None |
| 114 | with pytest.raises(RuntimeError, match="No storage backend configured"): |
| 115 | _get_backend_impl() |
| 116 | |
| 117 | def test_returns_blob_backend_when_blob_storage_bucket_set(self) -> None: |
| 118 | from musehub.storage.backends import BlobBackend, get_backend |
| 119 | with patch("musehub.storage.backends.settings") as mock_settings: |
| 120 | mock_settings.blob_storage_bucket = "muse-objects" |
| 121 | mock_settings.blob_storage_endpoint = MINIO_ENDPOINT |
| 122 | mock_settings.blob_storage_access_key_id = MINIO_ACCESS_KEY |
| 123 | mock_settings.blob_storage_secret_access_key = MINIO_SECRET_KEY |
| 124 | mock_settings.blob_storage_region = "auto" |
| 125 | result = get_backend() |
| 126 | assert isinstance(result, BlobBackend) |
| 127 | |
| 128 | def test_returns_blob_backend_when_aws_bucket_set(self) -> None: |
| 129 | from musehub.storage.backends import BlobBackend, get_backend |
| 130 | with patch("musehub.storage.backends.settings") as mock_settings: |
| 131 | mock_settings.blob_storage_bucket = None |
| 132 | mock_settings.aws_s3_asset_bucket = "my-bucket" |
| 133 | mock_settings.aws_region = "us-east-1" |
| 134 | mock_settings.blob_storage_endpoint = None |
| 135 | mock_settings.blob_storage_access_key_id = None |
| 136 | mock_settings.blob_storage_secret_access_key = None |
| 137 | result = get_backend() |
| 138 | assert isinstance(result, BlobBackend) |
| 139 | |
| 140 | def test_local_backend_is_deleted(self) -> None: |
| 141 | """LocalBackend must not exist in the module after deletion.""" |
| 142 | import musehub.storage.backends as mod |
| 143 | assert not hasattr(mod, "LocalBackend"), ( |
| 144 | "LocalBackend still exists in musehub.storage.backends — delete it" |
| 145 | ) |
| 146 | |
| 147 | def test_local_backend_not_in_storage_init(self) -> None: |
| 148 | """LocalBackend must not be exported from musehub.storage.""" |
| 149 | import musehub.storage as mod |
| 150 | assert not hasattr(mod, "LocalBackend"), ( |
| 151 | "LocalBackend still exported from musehub.storage — remove it from __all__" |
| 152 | ) |
| 153 | |
| 154 | |
| 155 | # ═══════════════════════════════════════════════════════════════════════════════ |
| 156 | # Tier 2 — Integration: S3Backend against live MinIO |
| 157 | # ═══════════════════════════════════════════════════════════════════════════════ |
| 158 | |
| 159 | |
| 160 | class TestS3BackendMinIO: |
| 161 | @requires_minio |
| 162 | async def test_put_get_round_trip(self) -> None: |
| 163 | backend = _minio_backend() |
| 164 | oid = _oid() |
| 165 | data = b"hello minio " + secrets.token_bytes(32) |
| 166 | await backend.put(oid, data) |
| 167 | result = await backend.get(oid) |
| 168 | assert result == data |
| 169 | |
| 170 | @requires_minio |
| 171 | async def test_exists_false_before_put(self) -> None: |
| 172 | backend = _minio_backend() |
| 173 | oid = _oid() |
| 174 | assert await backend.exists(oid) is False |
| 175 | |
| 176 | @requires_minio |
| 177 | async def test_exists_true_after_put(self) -> None: |
| 178 | backend = _minio_backend() |
| 179 | oid = _oid() |
| 180 | await backend.put(oid, b"exists-check") |
| 181 | assert await backend.exists(oid) is True |
| 182 | |
| 183 | @requires_minio |
| 184 | async def test_put_is_idempotent(self) -> None: |
| 185 | backend = _minio_backend() |
| 186 | oid = _oid() |
| 187 | data = b"idempotent-data" |
| 188 | await backend.put(oid, data) |
| 189 | await backend.put(oid, data) |
| 190 | result = await backend.get(oid) |
| 191 | assert result == data |
| 192 | |
| 193 | @requires_minio |
| 194 | async def test_get_returns_none_for_missing(self) -> None: |
| 195 | backend = _minio_backend() |
| 196 | oid = _oid() |
| 197 | result = await backend.get(oid) |
| 198 | assert result is None |
| 199 | |
| 200 | @requires_minio |
| 201 | async def test_delete_removes_object(self) -> None: |
| 202 | backend = _minio_backend() |
| 203 | oid = _oid() |
| 204 | await backend.put(oid, b"to-be-deleted") |
| 205 | assert await backend.exists(oid) is True |
| 206 | await backend.delete(oid) |
| 207 | assert await backend.exists(oid) is False |
| 208 | |
| 209 | @requires_minio |
| 210 | async def test_uri_for_uses_s3_scheme(self) -> None: |
| 211 | backend = _minio_backend() |
| 212 | oid = _oid() |
| 213 | uri = backend.uri_for(oid) |
| 214 | assert uri.startswith(f"s3://{MINIO_BUCKET}/") |
| 215 | |
| 216 | @requires_minio |
| 217 | async def test_get_backend_with_minio_env_vars(self) -> None: |
| 218 | """get_backend() wired to MinIO env vars returns a working S3Backend.""" |
| 219 | from musehub.storage.backends import get_backend |
| 220 | with patch("musehub.storage.backends.settings") as mock_settings: |
| 221 | mock_settings.blob_storage_bucket = MINIO_BUCKET |
| 222 | mock_settings.blob_storage_endpoint = MINIO_ENDPOINT |
| 223 | mock_settings.blob_storage_access_key_id = MINIO_ACCESS_KEY |
| 224 | mock_settings.blob_storage_secret_access_key = MINIO_SECRET_KEY |
| 225 | mock_settings.blob_storage_region = "us-east-1" |
| 226 | backend = get_backend() |
| 227 | |
| 228 | oid = _oid() |
| 229 | await backend.put(oid, b"via-get-backend") |
| 230 | assert await backend.get(oid) == b"via-get-backend" |
| 231 | |
| 232 | |
| 233 | # ═══════════════════════════════════════════════════════════════════════════════ |
| 234 | # Tier 3 — Mpack content-addressability |
| 235 | # ═══════════════════════════════════════════════════════════════════════════════ |
| 236 | |
| 237 | def _mpack_key_for(data: bytes) -> str: |
| 238 | return "sha256:" + hashlib.sha256(data).hexdigest() |
| 239 | |
| 240 | |
| 241 | def _minio_backend_with_public_endpoint(): |
| 242 | """Backend with public endpoint set so presigned URLs use localhost:9000.""" |
| 243 | from musehub.storage.backends import BlobBackend |
| 244 | return BlobBackend( |
| 245 | bucket=MINIO_BUCKET, |
| 246 | endpoint_url=MINIO_ENDPOINT, |
| 247 | public_endpoint_url=MINIO_ENDPOINT, |
| 248 | access_key_id=MINIO_ACCESS_KEY, |
| 249 | secret_access_key=MINIO_SECRET_KEY, |
| 250 | region="us-east-1", |
| 251 | ) |
| 252 | |
| 253 | |
| 254 | class TestMpackContentAddressability: |
| 255 | """Every mpack written must be readable back with sha256(bytes) == key.""" |
| 256 | |
| 257 | @requires_minio |
| 258 | async def test_put_mpack_get_mpack_round_trip(self) -> None: |
| 259 | """put_mpack then get_mpack must return bytes whose sha256 matches the key.""" |
| 260 | backend = _minio_backend_with_public_endpoint() |
| 261 | data = b"MUSE" + secrets.token_bytes(128) |
| 262 | key = _mpack_key_for(data) |
| 263 | |
| 264 | await backend.put_mpack(key, data) |
| 265 | result = await backend.get_mpack(key) |
| 266 | |
| 267 | assert result is not None, "get_mpack returned None — object not found" |
| 268 | actual_key = _mpack_key_for(result) |
| 269 | assert actual_key == key, ( |
| 270 | f"Content-addressability violated: wrote key={key[:30]}… " |
| 271 | f"but read back sha256={actual_key[:30]}…" |
| 272 | ) |
| 273 | assert result == data |
| 274 | |
| 275 | @requires_minio |
| 276 | async def test_presign_put_get_mpack_round_trip(self) -> None: |
| 277 | """Presigned PUT then get_mpack must return bytes whose sha256 matches the key. |
| 278 | |
| 279 | This is the push path: client PUTs via presigned URL, server reads via get_mpack. |
| 280 | If the presigned URL key encoding differs from _mpack_key, this test catches it. |
| 281 | """ |
| 282 | backend = _minio_backend_with_public_endpoint() |
| 283 | data = b"MUSE" + secrets.token_bytes(128) |
| 284 | key = _mpack_key_for(data) |
| 285 | |
| 286 | upload_url = await backend.presign_mpack_put(key, ttl_seconds=300) |
| 287 | |
| 288 | # PUT directly to the presigned URL (simulates the muse push client) |
| 289 | req = urllib.request.Request( |
| 290 | upload_url, |
| 291 | data=data, |
| 292 | method="PUT", |
| 293 | headers={"Content-Type": "application/x-muse-pack"}, |
| 294 | ) |
| 295 | with urllib.request.urlopen(req) as resp: |
| 296 | assert resp.status == 200, f"presigned PUT failed: HTTP {resp.status}" |
| 297 | |
| 298 | result = await backend.get_mpack(key) |
| 299 | |
| 300 | assert result is not None, ( |
| 301 | "get_mpack returned None after presigned PUT — key mismatch between " |
| 302 | f"presigned URL and _mpack_key. upload_url={upload_url}" |
| 303 | ) |
| 304 | actual_key = _mpack_key_for(result) |
| 305 | assert actual_key == key, ( |
| 306 | f"Content-addressability violated after presigned PUT: " |
| 307 | f"wrote key={key[:30]}… but read back sha256={actual_key[:30]}…\n" |
| 308 | f"upload_url={upload_url}" |
| 309 | ) |
File History
2 commits
sha256:eead4146ec6c9905a097a89f8dbffa3d5c5e8ef9c1acd0e8a5b2a93b0084d273
Mpack content-addressability
Human
13 hours ago
sha256:ef10830ce231e0a20efcb0e2586cb879471247e916616e6fdd0d51df459e2595
fix: typing audit — 0 violations, 0 untyped defs across all…
Sonnet 4.6
minor
⚠
13 days ago