indices.py
python
sha256:cf6265cea8c21d9228d90dec13ef6ec2dab5103d466db9cc4590681832de4bf8
docs(KD-STAGING): sync governance after KD-6b DONE
Human
13 days ago
| 1 | """Optional local index layer for Muse repositories. |
| 2 | |
| 3 | Indexes live under ``.muse/indices/`` and are derived, versioned, and fully |
| 4 | rebuildable from the commit history. **No index is required for repository |
| 5 | correctness** — all commands work without them; indexes only accelerate |
| 6 | repeated queries. |
| 7 | |
| 8 | Available indexes |
| 9 | ----------------- |
| 10 | |
| 11 | ``symbol_history`` |
| 12 | Maps symbol addresses to their event timeline across all commits. |
| 13 | Enables O(1) ``muse symbol-log``, ``muse lineage``, and ``muse query-history`` |
| 14 | instead of O(commits × files) scans. |
| 15 | |
| 16 | ``hash_occurrence`` |
| 17 | Maps ``body_hash`` values to the list of symbol addresses that share them. |
| 18 | Enables O(1) ``muse clones`` and ``muse find-symbol hash=``. |
| 19 | |
| 20 | Storage format |
| 21 | -------------- |
| 22 | |
| 23 | Index files are stored as **msgpack** (``*.msgpack``) under ``.muse/indices/``. |
| 24 | msgpack is used instead of JSON because it is ~2× faster to serialize and |
| 25 | deserialize, and the files are internal — they are never directly read by a |
| 26 | human or an agent. All user-facing output (``--json`` flag) is still JSON |
| 27 | built from the deserialized in-memory structures. |
| 28 | |
| 29 | Writes are **atomic**: data is flushed to a ``.tmp`` sibling and then renamed |
| 30 | over the target file so a crash mid-write never corrupts an index. |
| 31 | |
| 32 | Rebuild |
| 33 | ------- |
| 34 | |
| 35 | Indexes are rebuilt by ``muse code index rebuild``. They can also be built |
| 36 | incrementally: the ``update_*_index`` functions accept an existing index |
| 37 | dict and patch it rather than rebuilding from scratch. |
| 38 | """ |
| 39 | |
| 40 | from __future__ import annotations |
| 41 | |
| 42 | import datetime |
| 43 | import logging |
| 44 | import pathlib |
| 45 | from typing import TypedDict |
| 46 | |
| 47 | import msgpack |
| 48 | |
| 49 | from muse._version import __version__ as _SCHEMA_VERSION |
| 50 | from muse.core._types import Metadata, MsgpackDict |
| 51 | from muse.core.store import MsgpackValue |
| 52 | |
| 53 | logger = logging.getLogger(__name__) |
| 54 | |
| 55 | type _EntryList = list[Metadata] |
| 56 | type _EntryMap = dict[str, _EntryList] |
| 57 | type _StringListMap = dict[str, list[str]] |
| 58 | |
| 59 | # --------------------------------------------------------------------------- |
| 60 | # Typed index entry shapes |
| 61 | # --------------------------------------------------------------------------- |
| 62 | |
| 63 | |
| 64 | class SymbolHistoryEntry: |
| 65 | """One event in a symbol's history timeline.""" |
| 66 | |
| 67 | __slots__ = ( |
| 68 | "commit_id", "committed_at", "op", |
| 69 | "content_id", "body_hash", "signature_id", |
| 70 | ) |
| 71 | |
| 72 | def __init__( |
| 73 | self, |
| 74 | commit_id: str, |
| 75 | committed_at: str, |
| 76 | op: str, |
| 77 | content_id: str, |
| 78 | body_hash: str, |
| 79 | signature_id: str, |
| 80 | ) -> None: |
| 81 | self.commit_id = commit_id |
| 82 | self.committed_at = committed_at |
| 83 | self.op = op |
| 84 | self.content_id = content_id |
| 85 | self.body_hash = body_hash |
| 86 | self.signature_id = signature_id |
| 87 | |
| 88 | def to_dict(self) -> Metadata: |
| 89 | """Serialise to a plain string-keyed, string-valued mapping.""" |
| 90 | return { |
| 91 | "commit_id": self.commit_id, |
| 92 | "committed_at": self.committed_at, |
| 93 | "op": self.op, |
| 94 | "content_id": self.content_id, |
| 95 | "body_hash": self.body_hash, |
| 96 | "signature_id": self.signature_id, |
| 97 | } |
| 98 | |
| 99 | @classmethod |
| 100 | def from_dict(cls, d: Metadata) -> "SymbolHistoryEntry": |
| 101 | """Construct from a typed string-keyed mapping (e.g. ``to_dict`` output).""" |
| 102 | return cls( |
| 103 | commit_id=d["commit_id"], |
| 104 | committed_at=d["committed_at"], |
| 105 | op=d["op"], |
| 106 | content_id=d["content_id"], |
| 107 | body_hash=d["body_hash"], |
| 108 | signature_id=d["signature_id"], |
| 109 | ) |
| 110 | |
| 111 | @classmethod |
| 112 | def from_msgpack(cls, d: MsgpackDict) -> "SymbolHistoryEntry": |
| 113 | """Deserialise from a raw msgpack mapping (all fields coerced to ``str``).""" |
| 114 | def _s(key: str) -> str: |
| 115 | v = d.get(key, "") |
| 116 | return v if isinstance(v, str) else "" |
| 117 | return cls( |
| 118 | commit_id=_s("commit_id"), |
| 119 | committed_at=_s("committed_at"), |
| 120 | op=_s("op"), |
| 121 | content_id=_s("content_id"), |
| 122 | body_hash=_s("body_hash"), |
| 123 | signature_id=_s("signature_id"), |
| 124 | ) |
| 125 | |
| 126 | |
| 127 | # --------------------------------------------------------------------------- |
| 128 | # On-disk document shapes (one TypedDict per index type) |
| 129 | # --------------------------------------------------------------------------- |
| 130 | |
| 131 | |
| 132 | class _SymbolHistoryDoc(TypedDict): |
| 133 | """Msgpack document layout for the ``symbol_history`` index file.""" |
| 134 | |
| 135 | schema_version: str |
| 136 | index: str |
| 137 | updated_at: str |
| 138 | entries: _EntryMap |
| 139 | |
| 140 | |
| 141 | class _HashOccurrenceDoc(TypedDict): |
| 142 | """Msgpack document layout for the ``hash_occurrence`` index file.""" |
| 143 | |
| 144 | schema_version: str |
| 145 | index: str |
| 146 | updated_at: str |
| 147 | entries: _StringListMap |
| 148 | |
| 149 | |
| 150 | # --------------------------------------------------------------------------- |
| 151 | # Index I/O helpers |
| 152 | # --------------------------------------------------------------------------- |
| 153 | |
| 154 | |
| 155 | def _indices_dir(root: pathlib.Path) -> pathlib.Path: |
| 156 | return root / ".muse" / "indices" |
| 157 | |
| 158 | |
| 159 | def _index_path(root: pathlib.Path, name: str) -> pathlib.Path: |
| 160 | """Return the msgpack file path for a named index.""" |
| 161 | return _indices_dir(root) / f"{name}.msgpack" |
| 162 | |
| 163 | |
| 164 | def _ensure_dir(root: pathlib.Path) -> None: |
| 165 | _indices_dir(root).mkdir(parents=True, exist_ok=True) |
| 166 | |
| 167 | |
| 168 | def _now_iso() -> str: |
| 169 | return datetime.datetime.now(datetime.timezone.utc).isoformat() |
| 170 | |
| 171 | |
| 172 | def _write_symbol_history_doc(path: pathlib.Path, doc: _SymbolHistoryDoc) -> None: |
| 173 | """Atomically serialise a :class:`_SymbolHistoryDoc` to *path*.""" |
| 174 | _write_bytes_atomic(path, msgpack.packb(doc, use_bin_type=True)) |
| 175 | |
| 176 | |
| 177 | def _write_hash_occurrence_doc(path: pathlib.Path, doc: _HashOccurrenceDoc) -> None: |
| 178 | """Atomically serialise a :class:`_HashOccurrenceDoc` to *path*.""" |
| 179 | _write_bytes_atomic(path, msgpack.packb(doc, use_bin_type=True)) |
| 180 | |
| 181 | |
| 182 | # --------------------------------------------------------------------------- |
| 183 | # Read safety constants (index-specific) |
| 184 | # --------------------------------------------------------------------------- |
| 185 | |
| 186 | # Index files grow with repository size and can reach hundreds of MiB for |
| 187 | # large repos. The cap is more generous than the commit/snapshot store while |
| 188 | # still bounding worst-case memory allocation before read_bytes() is called. |
| 189 | _MAX_INDEX_BYTES: int = 512 * 1024 * 1024 # 512 MiB |
| 190 | |
| 191 | _INDEX_MAX_STR_LEN: int = 1_048_576 # 1 MiB per string value |
| 192 | _INDEX_MAX_BIN_LEN: int = 1_048_576 # allow binary body hashes stored as bytes |
| 193 | _INDEX_MAX_ARRAY_LEN: int = 10_000_000 # large repos have millions of symbol events |
| 194 | _INDEX_MAX_MAP_LEN: int = 10_000_000 # same — hash_occurrence can be very wide |
| 195 | |
| 196 | |
| 197 | def _write_bytes_atomic(path: pathlib.Path, packed: bytes) -> None: |
| 198 | """Replace *path* with *packed* using a ``.tmp`` rename to guarantee atomicity.""" |
| 199 | tmp = path.with_suffix(".tmp") |
| 200 | try: |
| 201 | tmp.write_bytes(packed) |
| 202 | tmp.replace(path) |
| 203 | except OSError: |
| 204 | tmp.unlink(missing_ok=True) |
| 205 | raise |
| 206 | |
| 207 | |
| 208 | def _read_msgpack(path: pathlib.Path) -> MsgpackValue: |
| 209 | """Read and unpack an index msgpack file, enforcing size and per-value limits. |
| 210 | |
| 211 | Index files can grow into the hundreds of MiB for large repositories, so the |
| 212 | cap is more generous than the commit/snapshot store. The stat check still |
| 213 | fires *before* ``read_bytes()`` so a gigantic or adversarially crafted index |
| 214 | file never triggers an OOM during the read itself. |
| 215 | |
| 216 | Raises :exc:`OSError` if the file exceeds ``_MAX_INDEX_BYTES``. |
| 217 | """ |
| 218 | size = path.stat().st_size |
| 219 | if size > _MAX_INDEX_BYTES: |
| 220 | raise OSError( |
| 221 | f"Index file {path.name!r} is {size:,} bytes, exceeding the " |
| 222 | f"{_MAX_INDEX_BYTES // (1024 * 1024)} MiB read limit." |
| 223 | ) |
| 224 | result: MsgpackValue = msgpack.unpackb( |
| 225 | path.read_bytes(), |
| 226 | raw=False, |
| 227 | max_str_len=_INDEX_MAX_STR_LEN, |
| 228 | max_bin_len=_INDEX_MAX_BIN_LEN, |
| 229 | max_array_len=_INDEX_MAX_ARRAY_LEN, |
| 230 | max_map_len=_INDEX_MAX_MAP_LEN, |
| 231 | ) |
| 232 | return result |
| 233 | |
| 234 | |
| 235 | # --------------------------------------------------------------------------- |
| 236 | # Symbol history index |
| 237 | # --------------------------------------------------------------------------- |
| 238 | |
| 239 | |
| 240 | SymbolHistoryIndex = dict[str, list[SymbolHistoryEntry]] |
| 241 | |
| 242 | |
| 243 | def load_symbol_history(root: pathlib.Path) -> SymbolHistoryIndex: |
| 244 | """Load the symbol history index, returning an empty dict if absent.""" |
| 245 | path = _index_path(root, "symbol_history") |
| 246 | if not path.exists(): |
| 247 | return {} |
| 248 | try: |
| 249 | raw = _read_msgpack(path) |
| 250 | if not isinstance(raw, dict): |
| 251 | raise ValueError("top-level document is not a dict") |
| 252 | entries_raw = raw.get("entries") |
| 253 | if not isinstance(entries_raw, dict): |
| 254 | return {} |
| 255 | result: SymbolHistoryIndex = {} |
| 256 | for address, entry_list in entries_raw.items(): |
| 257 | if not isinstance(entry_list, list): |
| 258 | continue |
| 259 | entries: list[SymbolHistoryEntry] = [] |
| 260 | for item in entry_list: |
| 261 | if isinstance(item, dict): |
| 262 | entries.append(SymbolHistoryEntry.from_msgpack(item)) |
| 263 | result[address] = entries |
| 264 | return result |
| 265 | except Exception as exc: |
| 266 | logger.warning("⚠️ Corrupt symbol_history index: %s — returning empty", exc) |
| 267 | return {} |
| 268 | |
| 269 | |
| 270 | def save_symbol_history(root: pathlib.Path, index: SymbolHistoryIndex) -> None: |
| 271 | """Persist the symbol history index as msgpack (atomic write).""" |
| 272 | _ensure_dir(root) |
| 273 | path = _index_path(root, "symbol_history") |
| 274 | doc = _SymbolHistoryDoc( |
| 275 | schema_version=_SCHEMA_VERSION, |
| 276 | index="symbol_history", |
| 277 | updated_at=_now_iso(), |
| 278 | entries={ |
| 279 | addr: [e.to_dict() for e in evts] |
| 280 | for addr, evts in sorted(index.items()) |
| 281 | }, |
| 282 | ) |
| 283 | _write_symbol_history_doc(path, doc) |
| 284 | logger.debug("✅ Saved symbol_history index (%d addresses)", len(index)) |
| 285 | |
| 286 | |
| 287 | # --------------------------------------------------------------------------- |
| 288 | # Hash occurrence index |
| 289 | # --------------------------------------------------------------------------- |
| 290 | |
| 291 | |
| 292 | HashOccurrenceIndex = dict[str, list[str]] |
| 293 | |
| 294 | |
| 295 | def load_hash_occurrence(root: pathlib.Path) -> HashOccurrenceIndex: |
| 296 | """Load the hash occurrence index, returning an empty dict if absent.""" |
| 297 | path = _index_path(root, "hash_occurrence") |
| 298 | if not path.exists(): |
| 299 | return {} |
| 300 | try: |
| 301 | raw = _read_msgpack(path) |
| 302 | if not isinstance(raw, dict): |
| 303 | raise ValueError("top-level document is not a dict") |
| 304 | entries_raw = raw.get("entries") |
| 305 | if not isinstance(entries_raw, dict): |
| 306 | return {} |
| 307 | result: HashOccurrenceIndex = {} |
| 308 | for body_hash, addresses in entries_raw.items(): |
| 309 | if isinstance(addresses, list): |
| 310 | result[body_hash] = [a for a in addresses if isinstance(a, str)] |
| 311 | return result |
| 312 | except Exception as exc: |
| 313 | logger.warning("⚠️ Corrupt hash_occurrence index: %s — returning empty", exc) |
| 314 | return {} |
| 315 | |
| 316 | |
| 317 | def save_hash_occurrence(root: pathlib.Path, index: HashOccurrenceIndex) -> None: |
| 318 | """Persist the hash occurrence index as msgpack (atomic write).""" |
| 319 | _ensure_dir(root) |
| 320 | path = _index_path(root, "hash_occurrence") |
| 321 | doc = _HashOccurrenceDoc( |
| 322 | schema_version=_SCHEMA_VERSION, |
| 323 | index="hash_occurrence", |
| 324 | updated_at=_now_iso(), |
| 325 | entries={h: sorted(addrs) for h, addrs in sorted(index.items())}, |
| 326 | ) |
| 327 | _write_hash_occurrence_doc(path, doc) |
| 328 | logger.debug("✅ Saved hash_occurrence index (%d hashes)", len(index)) |
| 329 | |
| 330 | |
| 331 | # --------------------------------------------------------------------------- |
| 332 | # Index metadata |
| 333 | # --------------------------------------------------------------------------- |
| 334 | |
| 335 | |
| 336 | #: All index names known to this module. |
| 337 | KNOWN_INDEX_NAMES: tuple[str, ...] = ("symbol_history", "hash_occurrence") |
| 338 | |
| 339 | |
| 340 | class IndexInfoEntry(TypedDict): |
| 341 | """Status information for one local index — returned by :func:`index_info`.""" |
| 342 | |
| 343 | name: str |
| 344 | status: str # "present" | "absent" | "corrupt" |
| 345 | entries: int |
| 346 | updated_at: str | None |
| 347 | |
| 348 | |
| 349 | def index_info(root: pathlib.Path) -> list[IndexInfoEntry]: |
| 350 | """Return status information about all known indexes. |
| 351 | |
| 352 | ``entries`` is always an ``int``. Callers must not convert it. |
| 353 | """ |
| 354 | result: list[IndexInfoEntry] = [] |
| 355 | for name in KNOWN_INDEX_NAMES: |
| 356 | path = _index_path(root, name) |
| 357 | if path.exists(): |
| 358 | try: |
| 359 | raw = _read_msgpack(path) |
| 360 | if not isinstance(raw, dict): |
| 361 | raise ValueError("not a dict") |
| 362 | upd = raw.get("updated_at") |
| 363 | updated_at: str | None = upd if isinstance(upd, str) else None |
| 364 | entries_data = raw.get("entries") |
| 365 | entries = len(entries_data) if isinstance(entries_data, dict) else 0 |
| 366 | result.append(IndexInfoEntry( |
| 367 | name=name, |
| 368 | status="present", |
| 369 | updated_at=updated_at, |
| 370 | entries=entries, |
| 371 | )) |
| 372 | except Exception: |
| 373 | result.append(IndexInfoEntry( |
| 374 | name=name, status="corrupt", updated_at=None, entries=0, |
| 375 | )) |
| 376 | else: |
| 377 | result.append(IndexInfoEntry( |
| 378 | name=name, status="absent", updated_at=None, entries=0, |
| 379 | )) |
| 380 | return result |
| 381 | |
| 382 | |
| 383 | def purge_index(root: pathlib.Path, name: str) -> bool: |
| 384 | """Delete the on-disk file for the named index. |
| 385 | |
| 386 | Returns ``True`` if the file existed and was deleted, ``False`` if it was |
| 387 | already absent. Raises ``ValueError`` for unknown index names. |
| 388 | """ |
| 389 | if name not in KNOWN_INDEX_NAMES: |
| 390 | raise ValueError(f"Unknown index name '{name}'. Valid names: {', '.join(KNOWN_INDEX_NAMES)}") |
| 391 | path = _index_path(root, name) |
| 392 | try: |
| 393 | path.unlink() |
| 394 | logger.debug("🗑️ Purged index: %s", name) |
| 395 | return True |
| 396 | except FileNotFoundError: |
| 397 | return False |
File History
1 commit
sha256:cf6265cea8c21d9228d90dec13ef6ec2dab5103d466db9cc4590681832de4bf8
docs(KD-STAGING): sync governance after KD-6b DONE
Human
13 days ago