"""Object compression and delta encoding for MWP wire transfer. Tier 1 — zlib: every object is zlib-compressed before transmission. Tier 2 — delta+zlib: objects with a known base are delta-encoded, then zlib-compressed. Both tiers use only Python stdlib (zlib, struct) — no external dependencies. The server always decompresses/reconstructs; the client always compresses. """ from __future__ import annotations import struct import zlib # Minimum byte-run length to emit a COPY instruction rather than DATA. # 32 bytes covers most source-code line edits without excessive overhead. _COPY_THRESHOLD = 32 def compress_zlib(data: bytes) -> bytes: """Compress *data* with zlib at level 1 (speed-optimised). Level 1 gives ~60 % reduction on source text with negligible CPU overhead at the throughput rates of a pack push (hundreds of objects per second). """ return zlib.compress(data, level=1) def decompress_zlib(data: bytes) -> bytes: """Decompress zlib-compressed *data*. Raises :class:`zlib.error` on corrupt input — let it propagate so the server returns 400 to the client rather than storing garbage. """ return zlib.decompress(data) def compute_delta(base: bytes, target: bytes) -> bytes: """Delta-encode *target* relative to *base*. Returns a zlib-compressed instruction stream. Each instruction is one of: * ``COPY``: ``b'\\x00' + struct.pack('>II', offset, length)`` — copy *length* bytes from *base* starting at *offset*. * ``DATA``: ``b'\\x01' + struct.pack('>I', length) + data_bytes`` — emit *length* literal bytes verbatim. Algorithm: Build a hash table mapping every *_COPY_THRESHOLD*-byte chunk of *base* to its last position. Walk *target* looking for matching chunks; extend each match forward as far as possible, then emit a single COPY instruction. Unmatched bytes accumulate in a DATA buffer that is flushed whenever a COPY is emitted. Raises :class:`ValueError` when the compressed delta is not smaller than a plain zlib-compressed copy of *target*; the caller should fall back to Tier 1 in that case. Args: base: Previous version of the object (already stored on the remote). target: New version of the object to encode. Returns: Zlib-compressed binary instruction stream. Raises: ValueError: Delta is not smaller than zlib(*target*) — not profitable. """ # Degenerate case: empty base → single DATA instruction. if not base: raw = b"\x01" + struct.pack(">I", len(target)) + target return zlib.compress(raw, level=1) # Build hash table: chunk → last position in base. table: dict[bytes, int] = {} for i in range(len(base) - _COPY_THRESHOLD + 1): table[base[i : i + _COPY_THRESHOLD]] = i parts: list[bytes] = [] data_buf = bytearray() pos = 0 while pos < len(target): # Try to find a matching chunk starting at *pos*. if pos + _COPY_THRESHOLD <= len(target): chunk = target[pos : pos + _COPY_THRESHOLD] base_pos = table.get(chunk) if base_pos is not None: # Extend match forward as far as bytes agree. length = _COPY_THRESHOLD while ( pos + length < len(target) and base_pos + length < len(base) and target[pos + length] == base[base_pos + length] ): length += 1 # Flush accumulated DATA before emitting COPY. if data_buf: parts.append( b"\x01" + struct.pack(">I", len(data_buf)) + bytes(data_buf) ) data_buf = bytearray() parts.append(b"\x00" + struct.pack(">II", base_pos, length)) pos += length continue # No match — accumulate literal byte. data_buf.append(target[pos]) pos += 1 # Flush trailing DATA. if data_buf: parts.append(b"\x01" + struct.pack(">I", len(data_buf)) + bytes(data_buf)) delta = zlib.compress(b"".join(parts), level=1) # Reject unprofitable deltas so the caller can fall back to plain zlib. plain = zlib.compress(target, level=1) if len(delta) >= len(plain): raise ValueError( f"delta ({len(delta)} B) not smaller than zlib({len(plain)} B) — not profitable" ) return delta def apply_delta(base: bytes, delta_compressed: bytes) -> bytes: """Reconstruct the target by applying *delta_compressed* to *base*. Inverse of :func:`compute_delta`. Parses the instruction stream produced by that function and assembles the target byte-for-byte. Args: base: The base object bytes (fetched from storage). delta_compressed: Zlib-compressed instruction stream. Returns: Reconstructed target bytes. Raises: zlib.error: *delta_compressed* is corrupt. ValueError: Unknown instruction type in stream. struct.error: Truncated instruction stream. """ stream = zlib.decompress(delta_compressed) result = bytearray() pos = 0 n = len(stream) while pos < n: instr_type = stream[pos] pos += 1 if instr_type == 0: # COPY offset, length = struct.unpack_from(">II", stream, pos) pos += 8 result.extend(base[offset : offset + length]) elif instr_type == 1: # DATA (length,) = struct.unpack_from(">I", stream, pos) pos += 4 result.extend(stream[pos : pos + length]) pos += length else: raise ValueError(f"unknown delta instruction type: {instr_type:#04x} at stream offset {pos - 1}") return bytes(result)