index_rebuild.py
python
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6
chore(tests): add docstring to tests/__init__.py so rc14 tr…
Human
32 days ago
| 1 | """muse code index — manage and rebuild the optional local index layer. |
| 2 | |
| 3 | Indexes live under ``.muse/indices/`` and are fully derived from the commit |
| 4 | history. They are optional — all commands work without them, but indexes |
| 5 | dramatically accelerate repeated queries on large repositories. |
| 6 | |
| 7 | Available indexes |
| 8 | ----------------- |
| 9 | |
| 10 | ``symbol_history`` |
| 11 | Maps every symbol address to its full event timeline across all commits. |
| 12 | Reduces ``muse code symbol-log``, ``muse code lineage``, and |
| 13 | ``muse code query-history`` from O(commits × files) to O(1) lookups. |
| 14 | |
| 15 | ``hash_occurrence`` |
| 16 | Maps every ``body_hash`` to the list of addresses that share it. |
| 17 | Reduces ``muse code clones`` and ``muse code find-symbol hash=`` to O(1). |
| 18 | |
| 19 | Sub-commands |
| 20 | ------------ |
| 21 | |
| 22 | ``muse code index status`` |
| 23 | Show the status, entry count, and last-updated time of each index. |
| 24 | |
| 25 | ``muse code index rebuild [--index NAME] [--dry-run]`` |
| 26 | Rebuild one or all indexes by walking the entire commit history. |
| 27 | Safe to run multiple times. Pass ``--dry-run`` to see what would be |
| 28 | built without writing anything. |
| 29 | |
| 30 | ``muse code index purge [--index NAME]`` |
| 31 | Delete one or all local index files. The next rebuild recreates them. |
| 32 | |
| 33 | Usage:: |
| 34 | |
| 35 | muse code index status |
| 36 | muse code index status --json |
| 37 | muse code index rebuild |
| 38 | muse code index rebuild --json |
| 39 | muse code index rebuild --index symbol_history |
| 40 | muse code index rebuild --index hash_occurrence |
| 41 | muse code index rebuild --dry-run |
| 42 | muse code index purge |
| 43 | muse code index purge --index symbol_history |
| 44 | |
| 45 | JSON output — ``muse code index status --json``:: |
| 46 | |
| 47 | [ |
| 48 | {"name": "symbol_history", "status": "present", "entries": 1024, |
| 49 | "updated_at": "2026-03-21T12:00:00+00:00"}, |
| 50 | {"name": "hash_occurrence", "status": "absent", "entries": 0, |
| 51 | "updated_at": null} |
| 52 | ] |
| 53 | |
| 54 | JSON output — ``muse code index rebuild --json``:: |
| 55 | |
| 56 | {"schema_version": "0.1.5", |
| 57 | "rebuilt": ["symbol_history", "hash_occurrence"], |
| 58 | "symbol_history_addresses": 512, "symbol_history_events": 2048, |
| 59 | "hash_occurrence_clusters": 31, "hash_occurrence_addresses": 87} |
| 60 | """ |
| 61 | |
| 62 | from __future__ import annotations |
| 63 | |
| 64 | import argparse |
| 65 | import json |
| 66 | import logging |
| 67 | import pathlib |
| 68 | import sys |
| 69 | from typing import TypedDict |
| 70 | |
| 71 | from muse import __version__ |
| 72 | from muse.core.errors import ExitCode |
| 73 | from muse.core.indices import ( |
| 74 | KNOWN_INDEX_NAMES, |
| 75 | HashOccurrenceIndex, |
| 76 | IndexInfoEntry, |
| 77 | SymbolHistoryEntry, |
| 78 | SymbolHistoryIndex, |
| 79 | index_info, |
| 80 | purge_index, |
| 81 | save_hash_occurrence, |
| 82 | save_symbol_history, |
| 83 | ) |
| 84 | from muse.core.object_store import read_object |
| 85 | from muse.core.repo import require_repo |
| 86 | from muse.core.store import get_all_commits, get_commit_snapshot_manifest, read_current_branch |
| 87 | from muse.core.symbol_cache import SymbolCache, load_symbol_cache |
| 88 | from muse.plugins.code._query import is_semantic |
| 89 | from muse.plugins.code.ast_parser import parse_symbols |
| 90 | from muse.core.validation import sanitize_display |
| 91 | |
| 92 | |
| 93 | |
| 94 | type _BlobCache = dict[str, bytes] |
| 95 | type _ManifestCache = dict[str, dict[str, str]] |
| 96 | logger = logging.getLogger(__name__) |
| 97 | |
| 98 | |
| 99 | # --------------------------------------------------------------------------- |
| 100 | # TypedDict for rebuild JSON output schema |
| 101 | # --------------------------------------------------------------------------- |
| 102 | |
| 103 | |
| 104 | class _RebuildResult(TypedDict, total=False): |
| 105 | schema_version: str |
| 106 | dry_run: bool |
| 107 | rebuilt: list[str] |
| 108 | symbol_history_addresses: int |
| 109 | symbol_history_events: int |
| 110 | hash_occurrence_clusters: int |
| 111 | hash_occurrence_addresses: int |
| 112 | # Knowtation domain extension (Phase 3.4) |
| 113 | knowtation_link_graph_notes: int |
| 114 | knowtation_link_graph_edges_resolved: int |
| 115 | knowtation_link_graph_edges_broken: int |
| 116 | knowtation_link_graph_entities: int |
| 117 | |
| 118 | |
| 119 | # --------------------------------------------------------------------------- |
| 120 | # Index build logic |
| 121 | # --------------------------------------------------------------------------- |
| 122 | |
| 123 | |
| 124 | def _build_symbol_history( |
| 125 | root: pathlib.Path, |
| 126 | symbol_cache: SymbolCache | None = None, |
| 127 | ) -> SymbolHistoryIndex: |
| 128 | """Walk all commits oldest-first and build the symbol history index. |
| 129 | |
| 130 | Performance notes |
| 131 | ----------------- |
| 132 | * A ``manifest_cache`` (keyed by commit_id) ensures that each snapshot |
| 133 | manifest is fetched at most once per rebuild, regardless of how many |
| 134 | symbol ops share the same commit. |
| 135 | |
| 136 | * A ``blob_cache`` (keyed by obj_id) ensures that each source blob is |
| 137 | ``read_object``'d at most once per rebuild. |
| 138 | |
| 139 | * ``symbol_cache`` is the **persistent** cross-run parse cache |
| 140 | (:class:`~muse.core.symbol_cache.SymbolCache`). Because obj_id is the |
| 141 | SHA-256 of file bytes (content-addressed), a file that did not change |
| 142 | between two commits always produces a cache hit — its AST is never |
| 143 | re-parsed. On a warm cache a 300-commit rebuild drops from minutes to |
| 144 | seconds. |
| 145 | |
| 146 | Together these three layers reduce I/O from |
| 147 | O(commits × ops × files) on the first run to |
| 148 | O(1) per unique (obj_id, address) pair on subsequent runs. |
| 149 | """ |
| 150 | all_commits = sorted( |
| 151 | get_all_commits(root), |
| 152 | key=lambda c: c.committed_at, |
| 153 | ) |
| 154 | index: SymbolHistoryIndex = {} |
| 155 | |
| 156 | # manifest_cache: commit_id → {file_path: obj_id} |
| 157 | manifest_cache: _ManifestCache = {} |
| 158 | # blob_cache: obj_id → raw bytes (within this run; SymbolCache handles cross-run) |
| 159 | blob_cache: _BlobCache = {} |
| 160 | |
| 161 | for commit in all_commits: |
| 162 | if commit.structured_delta is None: |
| 163 | continue |
| 164 | committed_at = commit.committed_at.isoformat() |
| 165 | ops = commit.structured_delta.get("ops", []) |
| 166 | |
| 167 | # Fetch the manifest once per commit, not once per child op. |
| 168 | if commit.commit_id not in manifest_cache: |
| 169 | raw_manifest = get_commit_snapshot_manifest(root, commit.commit_id) |
| 170 | if raw_manifest is None: |
| 171 | logger.debug( |
| 172 | "Missing snapshot manifest for commit %s — skipping", |
| 173 | commit.commit_id[:8], |
| 174 | ) |
| 175 | continue |
| 176 | manifest_cache[commit.commit_id] = raw_manifest |
| 177 | manifest = manifest_cache[commit.commit_id] |
| 178 | |
| 179 | for op in ops: |
| 180 | if op["op"] != "patch": |
| 181 | continue |
| 182 | for child in op.get("child_ops", []): |
| 183 | addr = child["address"] |
| 184 | if "::" not in addr: |
| 185 | continue |
| 186 | file_path = addr.split("::")[0] |
| 187 | if not is_semantic(file_path): |
| 188 | continue |
| 189 | child_op = child["op"] |
| 190 | if child_op not in ("insert", "delete", "replace"): |
| 191 | continue |
| 192 | |
| 193 | # Extract hash fields from the snapshot blob. |
| 194 | # Resolution order: |
| 195 | # 1. symbol_cache (persistent msgpack cache, keyed by obj_id) |
| 196 | # 2. blob_cache (in-run bytes cache, avoids duplicate reads) |
| 197 | # 3. read_object + parse_symbols (cache miss — first encounter) |
| 198 | obj_id = manifest.get(file_path) |
| 199 | body_hash = "" |
| 200 | signature_id = "" |
| 201 | content_id = "" |
| 202 | if obj_id: |
| 203 | # Try the persistent SymbolCache first. |
| 204 | tree = symbol_cache.get(obj_id) if symbol_cache else None |
| 205 | if tree is None: |
| 206 | # Fetch bytes (in-run blob_cache avoids duplicate reads). |
| 207 | if obj_id not in blob_cache: |
| 208 | raw = read_object(root, obj_id) |
| 209 | if raw is not None: |
| 210 | blob_cache[obj_id] = raw |
| 211 | blob = blob_cache.get(obj_id) |
| 212 | if blob is not None: |
| 213 | tree = parse_symbols(blob, file_path) |
| 214 | if symbol_cache is not None: |
| 215 | symbol_cache.put(obj_id, tree) |
| 216 | if tree is not None: |
| 217 | rec = tree.get(addr) |
| 218 | if rec: |
| 219 | body_hash = rec["body_hash"] |
| 220 | signature_id = rec["signature_id"] |
| 221 | content_id = rec["content_id"] |
| 222 | |
| 223 | if not content_id: |
| 224 | # Fall back to the content_id stored in the delta itself. |
| 225 | if child_op == "insert": |
| 226 | content_id = str(child.get("content_id") or "") |
| 227 | elif child_op == "delete": |
| 228 | content_id = str(child.get("content_id") or "") |
| 229 | elif child_op == "replace": |
| 230 | content_id = str(child.get("new_content_id") or "") |
| 231 | |
| 232 | entry = SymbolHistoryEntry( |
| 233 | commit_id=commit.commit_id, |
| 234 | committed_at=committed_at, |
| 235 | op=child_op, |
| 236 | content_id=content_id, |
| 237 | body_hash=body_hash, |
| 238 | signature_id=signature_id, |
| 239 | ) |
| 240 | index.setdefault(addr, []).append(entry) |
| 241 | |
| 242 | return index |
| 243 | |
| 244 | |
| 245 | def _build_hash_occurrence(root: pathlib.Path) -> HashOccurrenceIndex: |
| 246 | """Walk the HEAD snapshot and build the hash occurrence index.""" |
| 247 | try: |
| 248 | muse_dir = root / ".muse" |
| 249 | branch = read_current_branch(root) |
| 250 | head_commit_id = (muse_dir / "refs" / "heads" / branch).read_text().strip() |
| 251 | except OSError as exc: |
| 252 | logger.debug("Could not determine HEAD commit for hash_occurrence build: %s", exc) |
| 253 | return {} |
| 254 | |
| 255 | raw_manifest = get_commit_snapshot_manifest(root, head_commit_id) |
| 256 | if raw_manifest is None: |
| 257 | logger.debug( |
| 258 | "Missing snapshot manifest for HEAD %s — hash_occurrence will be empty", |
| 259 | head_commit_id[:8], |
| 260 | ) |
| 261 | return {} |
| 262 | |
| 263 | index: HashOccurrenceIndex = {} |
| 264 | for file_path, obj_id in sorted(raw_manifest.items()): |
| 265 | if not is_semantic(file_path): |
| 266 | continue |
| 267 | raw = read_object(root, obj_id) |
| 268 | if raw is None: |
| 269 | continue |
| 270 | tree = parse_symbols(raw, file_path) |
| 271 | for addr, rec in tree.items(): |
| 272 | if rec["kind"] == "import": |
| 273 | continue |
| 274 | bh = rec["body_hash"] |
| 275 | index.setdefault(bh, []).append(addr) |
| 276 | |
| 277 | # Remove trivial (size-1) entries — they are not clones. |
| 278 | return {h: addrs for h, addrs in index.items() if len(addrs) > 1} |
| 279 | |
| 280 | |
| 281 | # --------------------------------------------------------------------------- |
| 282 | # Sub-commands |
| 283 | # --------------------------------------------------------------------------- |
| 284 | |
| 285 | |
| 286 | def register(subparsers: "argparse._SubParsersAction[argparse.ArgumentParser]") -> None: |
| 287 | """Register the index subcommand.""" |
| 288 | parser = subparsers.add_parser( |
| 289 | "index", |
| 290 | help="Manage the optional local index layer.", |
| 291 | description=__doc__, |
| 292 | formatter_class=argparse.RawDescriptionHelpFormatter, |
| 293 | ) |
| 294 | subs = parser.add_subparsers(dest="subcommand", metavar="SUBCOMMAND") |
| 295 | subs.required = True |
| 296 | |
| 297 | # --- purge --- |
| 298 | purge_p = subs.add_parser( |
| 299 | "purge", |
| 300 | help="Delete one or all local index files.", |
| 301 | description=( |
| 302 | "Delete index files under .muse/indices/. The canonical commit\n" |
| 303 | "history and object store are never touched — indexes are fully\n" |
| 304 | "rebuildable at any time with 'muse code index rebuild'.\n" |
| 305 | "Absent indexes are silently skipped (exit 0).\n\n" |
| 306 | "Agent quickstart\n" |
| 307 | "----------------\n" |
| 308 | " muse code index purge --json\n" |
| 309 | " muse code index purge -j\n" |
| 310 | " muse code index purge -j | jq .purged\n" |
| 311 | " muse code index purge --index symbol_history -j\n\n" |
| 312 | "JSON output schema\n" |
| 313 | "------------------\n" |
| 314 | ' {"schema_version": "<str>",\n' |
| 315 | ' "purged": ["symbol_history", ...],\n' |
| 316 | ' "skipped": ["hash_occurrence", ...]}\n\n' |
| 317 | "Exit codes\n" |
| 318 | "----------\n" |
| 319 | " 0 — operation completed (skipped indexes do not cause failure)\n" |
| 320 | " 2 — not inside a Muse repository\n" |
| 321 | ), |
| 322 | formatter_class=argparse.RawDescriptionHelpFormatter, |
| 323 | ) |
| 324 | purge_p.add_argument( |
| 325 | "--index", "-i", |
| 326 | dest="index_name", |
| 327 | default=None, |
| 328 | metavar="NAME", |
| 329 | choices=list(KNOWN_INDEX_NAMES), |
| 330 | help="Purge a specific index. Default: purge all.", |
| 331 | ) |
| 332 | purge_p.add_argument("--json", "-j", dest="as_json", action="store_true", |
| 333 | help="Emit purge summary as JSON.") |
| 334 | purge_p.set_defaults(func=run_purge) |
| 335 | |
| 336 | # --- rebuild --- |
| 337 | rebuild_p = subs.add_parser( |
| 338 | "rebuild", |
| 339 | help="Rebuild local indexes from the full commit history.", |
| 340 | description=( |
| 341 | "Rebuild symbol_history and/or hash_occurrence under .muse/indices/.\n" |
| 342 | "Safe to run any number of times — atomic writes prevent corruption.\n" |
| 343 | "A warm SymbolCache means unchanged files are never re-parsed.\n\n" |
| 344 | "Use --dry-run to compute statistics without writing to disk.\n\n" |
| 345 | "Agent quickstart\n" |
| 346 | "----------------\n" |
| 347 | " muse code index rebuild --json\n" |
| 348 | " muse code index rebuild -j\n" |
| 349 | " muse code index rebuild -j | jq .rebuilt\n" |
| 350 | " muse code index rebuild --index symbol_history -j\n" |
| 351 | " muse code index rebuild --dry-run -j\n\n" |
| 352 | "JSON output schema\n" |
| 353 | "------------------\n" |
| 354 | ' {"schema_version": "<str>", "dry_run": <bool>,\n' |
| 355 | ' "rebuilt": ["symbol_history", "hash_occurrence"],\n' |
| 356 | ' "symbol_history_addresses": <int>, "symbol_history_events": <int>,\n' |
| 357 | ' "hash_occurrence_clusters": <int>, "hash_occurrence_addresses": <int>}\n\n' |
| 358 | " Note: *_addresses / *_events / *_clusters keys are absent when the\n" |
| 359 | " corresponding index was not rebuilt.\n\n" |
| 360 | "Exit codes\n" |
| 361 | "----------\n" |
| 362 | " 0 — rebuild (or dry run) completed successfully\n" |
| 363 | " 2 — not inside a Muse repository\n" |
| 364 | ), |
| 365 | formatter_class=argparse.RawDescriptionHelpFormatter, |
| 366 | ) |
| 367 | rebuild_p.add_argument( |
| 368 | "--index", "-i", |
| 369 | dest="index_name", |
| 370 | default=None, |
| 371 | metavar="NAME", |
| 372 | choices=list(KNOWN_INDEX_NAMES), |
| 373 | help="Rebuild a specific index. Default: rebuild all.", |
| 374 | ) |
| 375 | rebuild_p.add_argument("--dry-run", action="store_true", |
| 376 | help="Compute what would be built without writing anything.") |
| 377 | rebuild_p.add_argument("--verbose", "-v", action="store_true", help="Show progress.") |
| 378 | rebuild_p.add_argument("--json", "-j", dest="as_json", action="store_true", |
| 379 | help="Emit rebuild summary as JSON.") |
| 380 | rebuild_p.set_defaults(func=run_rebuild) |
| 381 | |
| 382 | # --- status --- |
| 383 | status_p = subs.add_parser( |
| 384 | "status", |
| 385 | help="Show the status and entry count of each local index.", |
| 386 | description=( |
| 387 | "Report the on-disk state of every index under .muse/indices/.\n" |
| 388 | "Each index is either present (valid), absent (not yet built),\n" |
| 389 | "or corrupt (exists but failed to parse).\n\n" |
| 390 | "Agent quickstart\n" |
| 391 | "----------------\n" |
| 392 | " muse code index status --json\n" |
| 393 | " muse code index status -j\n" |
| 394 | " muse code index status -j | jq '.[].status'\n" |
| 395 | " muse code index status -j | jq '.[] | select(.status != \"present\")'\n\n" |
| 396 | "JSON output schema\n" |
| 397 | "------------------\n" |
| 398 | " [{\"name\": \"symbol_history\", \"status\": \"present|absent|corrupt\",\n" |
| 399 | ' "entries": <int>, "updated_at": "<iso8601>|null"}, ...]\n\n' |
| 400 | "Exit codes\n" |
| 401 | "----------\n" |
| 402 | " 0 — status emitted (absent/corrupt indexes do NOT cause non-zero exit)\n" |
| 403 | " 2 — not inside a Muse repository\n" |
| 404 | ), |
| 405 | formatter_class=argparse.RawDescriptionHelpFormatter, |
| 406 | ) |
| 407 | status_p.add_argument("--json", "-j", dest="as_json", action="store_true", help="Emit index status as JSON.") |
| 408 | status_p.set_defaults(func=run_status) |
| 409 | |
| 410 | |
| 411 | def run_status(args: argparse.Namespace) -> None: |
| 412 | """Show the status and entry count of each local Muse index. |
| 413 | |
| 414 | Reports the on-disk state of every index under ``.muse/indices/``. |
| 415 | Each index is reported as ``present`` (readable and valid), ``absent`` |
| 416 | (file not found — not yet built), or ``corrupt`` (file exists but |
| 417 | failed to parse). |
| 418 | |
| 419 | Indexes are derived entirely from commit history and can be rebuilt at |
| 420 | any time without modifying the canonical store. |
| 421 | |
| 422 | Security: all text-mode output passes through ``sanitize_display()`` so |
| 423 | index names cannot inject ANSI or control characters into the terminal. |
| 424 | Index names in JSON output come exclusively from ``KNOWN_INDEX_NAMES`` |
| 425 | (a static compile-time tuple), not from user input. The ``updated_at`` |
| 426 | and ``entries`` values are read from trusted local msgpack files in |
| 427 | ``.muse/indices/``; they are not derived from user-supplied data. |
| 428 | |
| 429 | JSON output fields (``--json`` / ``-j``) |
| 430 | ----------------------------------------- |
| 431 | Returns a JSON array, one object per known index: |
| 432 | |
| 433 | ``name`` |
| 434 | Registry key for the index (``"symbol_history"`` or |
| 435 | ``"hash_occurrence"``). |
| 436 | ``status`` |
| 437 | ``"present"`` — built and readable. |
| 438 | ``"absent"`` — not yet built; run ``muse code index rebuild``. |
| 439 | ``"corrupt"`` — file exists but failed to parse; run rebuild. |
| 440 | ``entries`` |
| 441 | Number of top-level entries in the index (``0`` when absent or |
| 442 | corrupt). |
| 443 | ``updated_at`` |
| 444 | ISO-8601 timestamp of the last rebuild, or ``null``. |
| 445 | |
| 446 | Exit codes |
| 447 | ---------- |
| 448 | 0 — status emitted (even if indexes are absent or corrupt) |
| 449 | 2 — not inside a Muse repository |
| 450 | """ |
| 451 | as_json: bool = args.as_json |
| 452 | |
| 453 | root = require_repo() |
| 454 | muse_dir = root / ".muse" |
| 455 | infos: list[IndexInfoEntry] = index_info(root) |
| 456 | |
| 457 | if as_json: |
| 458 | out = [ |
| 459 | { |
| 460 | "name": info["name"], |
| 461 | "status": info["status"], |
| 462 | "entries": info["entries"], |
| 463 | "updated_at": info["updated_at"], |
| 464 | } |
| 465 | for info in infos |
| 466 | ] |
| 467 | print(json.dumps(out)) |
| 468 | return |
| 469 | |
| 470 | print("\nLocal index status:") |
| 471 | print("─" * 50) |
| 472 | for info in infos: |
| 473 | status = info["status"] |
| 474 | name = info["name"] |
| 475 | updated = (info["updated_at"] or "")[:19] |
| 476 | entries = info["entries"] |
| 477 | if status == "present": |
| 478 | print(f" ✅ {sanitize_display(name):<20} {entries:>8} entries (updated {updated})") |
| 479 | elif status == "absent": |
| 480 | print(f" ⬜ {sanitize_display(name):<20} (not built — run: muse code index rebuild)") |
| 481 | else: |
| 482 | print(f" ❌ {sanitize_display(name):<20} corrupt — run: muse code index rebuild") |
| 483 | print() |
| 484 | |
| 485 | |
| 486 | def run_rebuild(args: argparse.Namespace) -> None: |
| 487 | """Rebuild local indexes from the full commit history. |
| 488 | |
| 489 | Walks all commits oldest-first to build ``symbol_history`` and/or |
| 490 | ``hash_occurrence`` under ``.muse/indices/``. Safe to run any number of |
| 491 | times — existing index files are atomically overwritten. |
| 492 | |
| 493 | A shared ``SymbolCache`` (keyed by content-addressed object ID) means AST |
| 494 | parses are never repeated for unchanged files. On a warm cache a |
| 495 | 300-commit rebuild drops from minutes to seconds. |
| 496 | |
| 497 | With ``--dry-run`` the full build is performed in memory and statistics |
| 498 | are reported, but nothing is written to disk and the symbol cache is not |
| 499 | saved. |
| 500 | |
| 501 | Security: the ``--index`` argument is validated by argparse against |
| 502 | ``KNOWN_INDEX_NAMES`` before ``run_rebuild`` is called — no |
| 503 | caller-supplied name reaches the file system unvalidated. Writes use |
| 504 | atomic msgpack saves (``save_symbol_history`` / ``save_hash_occurrence``) |
| 505 | so a crash cannot corrupt existing index data. |
| 506 | |
| 507 | JSON output fields (``--json`` / ``-j``) |
| 508 | ----------------------------------------- |
| 509 | ``schema_version`` |
| 510 | Muse version string at build time. |
| 511 | ``dry_run`` |
| 512 | ``true`` when ``--dry-run`` was passed; no files were written. |
| 513 | ``rebuilt`` |
| 514 | List of index names that were (or would be) rebuilt. |
| 515 | ``symbol_history_addresses`` |
| 516 | Number of distinct symbol addresses in the new index. |
| 517 | Present only when ``symbol_history`` was rebuilt. |
| 518 | ``symbol_history_events`` |
| 519 | Total number of insert/delete/replace events across all addresses. |
| 520 | Present only when ``symbol_history`` was rebuilt. |
| 521 | ``hash_occurrence_clusters`` |
| 522 | Number of hash clusters with more than one address (clone groups). |
| 523 | Present only when ``hash_occurrence`` was rebuilt. |
| 524 | ``hash_occurrence_addresses`` |
| 525 | Total address count across all clone clusters. |
| 526 | Present only when ``hash_occurrence`` was rebuilt. |
| 527 | |
| 528 | Exit codes |
| 529 | ---------- |
| 530 | 0 — rebuild completed (or dry run computed) successfully |
| 531 | 2 — not inside a Muse repository |
| 532 | """ |
| 533 | index_name: str | None = args.index_name |
| 534 | dry_run: bool = args.dry_run |
| 535 | verbose: bool = args.verbose |
| 536 | as_json: bool = args.as_json |
| 537 | |
| 538 | root = require_repo() |
| 539 | muse_dir = root / ".muse" |
| 540 | |
| 541 | build_all = index_name is None |
| 542 | built: list[str] = [] |
| 543 | result: _RebuildResult = { |
| 544 | "schema_version": __version__, |
| 545 | "dry_run": dry_run, |
| 546 | } |
| 547 | |
| 548 | # Load the persistent SymbolCache once — it is shared across both index |
| 549 | # builds and saved at the end. This gives cross-run caching of AST parses |
| 550 | # so that unchanged files are never re-parsed. |
| 551 | sym_cache = load_symbol_cache(root) |
| 552 | |
| 553 | if build_all or index_name == "symbol_history": |
| 554 | if verbose and not as_json: |
| 555 | print("Building symbol_history index…") |
| 556 | idx = _build_symbol_history(root, symbol_cache=sym_cache) |
| 557 | if not dry_run: |
| 558 | save_symbol_history(root, idx) |
| 559 | n_events = sum(len(evts) for evts in idx.values()) |
| 560 | result["symbol_history_addresses"] = len(idx) |
| 561 | result["symbol_history_events"] = n_events |
| 562 | if not as_json: |
| 563 | tag = " (dry run)" if dry_run else "" |
| 564 | print(f" ✅ symbol_history — {len(idx)} addresses, {n_events} events{tag}") |
| 565 | built.append("symbol_history") |
| 566 | |
| 567 | if build_all or index_name == "hash_occurrence": |
| 568 | if verbose and not as_json: |
| 569 | print("Building hash_occurrence index…") |
| 570 | idx2 = _build_hash_occurrence(root) |
| 571 | if not dry_run: |
| 572 | save_hash_occurrence(root, idx2) |
| 573 | n_clones = sum(len(addrs) for addrs in idx2.values()) |
| 574 | result["hash_occurrence_clusters"] = len(idx2) |
| 575 | result["hash_occurrence_addresses"] = n_clones |
| 576 | if not as_json: |
| 577 | tag = " (dry run)" if dry_run else "" |
| 578 | print(f" ✅ hash_occurrence — {len(idx2)} clone clusters, {n_clones} addresses{tag}") |
| 579 | built.append("hash_occurrence") |
| 580 | |
| 581 | # ── Knowtation-domain extension (Phase 3.4) ────────────────────────────── |
| 582 | # When the active domain is "knowtation", also rebuild the in-memory link |
| 583 | # graph (wikilinks + markdown refs + entity refs). This currently runs |
| 584 | # on-the-fly per CLI call (build_link_index < 10s on 5k notes), so this |
| 585 | # rebuild simply verifies it can be constructed cleanly and reports stats. |
| 586 | try: |
| 587 | from muse.plugins.registry import read_domain # noqa: PLC0415 |
| 588 | |
| 589 | active_domain = read_domain(root) |
| 590 | except Exception: # noqa: BLE001 — fall through to Code-domain rebuild |
| 591 | active_domain = "code" |
| 592 | |
| 593 | if active_domain == "knowtation": |
| 594 | try: |
| 595 | from muse.plugins.knowtation.link_index import build_link_index # noqa: PLC0415 |
| 596 | |
| 597 | if verbose and not as_json: |
| 598 | print("Building knowtation_link_graph index (in-memory verification)…") |
| 599 | link_idx = build_link_index(root) |
| 600 | n_resolved = sum( |
| 601 | 1 for links in link_idx.forward.values() |
| 602 | for link in links |
| 603 | if link.target is not None |
| 604 | ) |
| 605 | n_broken = len(link_idx.broken_links) |
| 606 | n_entities = len(link_idx.entity_index) |
| 607 | result["knowtation_link_graph_notes"] = link_idx.notes_indexed |
| 608 | result["knowtation_link_graph_edges_resolved"] = n_resolved |
| 609 | result["knowtation_link_graph_edges_broken"] = n_broken |
| 610 | result["knowtation_link_graph_entities"] = n_entities |
| 611 | built.append("knowtation_link_graph") |
| 612 | if not as_json: |
| 613 | tag = " (dry run)" if dry_run else "" |
| 614 | print( |
| 615 | f" ✅ knowtation_link_graph — {link_idx.notes_indexed} notes, " |
| 616 | f"{n_resolved} resolved edges, {n_broken} broken, " |
| 617 | f"{n_entities} entities{tag}" |
| 618 | ) |
| 619 | except Exception as exc: # noqa: BLE001 |
| 620 | logger.debug("knowtation index rebuild failed: %s", exc) |
| 621 | if not as_json: |
| 622 | print( |
| 623 | f" ⚠️ knowtation_link_graph — rebuild failed: {exc}", |
| 624 | file=sys.stderr, |
| 625 | ) |
| 626 | |
| 627 | result["rebuilt"] = built |
| 628 | |
| 629 | # Persist any newly cached parse results so the next rebuild is faster. |
| 630 | if not dry_run: |
| 631 | sym_cache.save() |
| 632 | |
| 633 | if as_json: |
| 634 | print(json.dumps(result)) |
| 635 | return |
| 636 | |
| 637 | action = "Computed" if dry_run else "Rebuilt" |
| 638 | print(f"\n{action} {len(built)} index(es)" + (" — no files written (dry run)" if dry_run else " under .muse/indices/")) |
| 639 | if not dry_run: |
| 640 | print("Run 'muse code index status' to verify.") |
| 641 | |
| 642 | |
| 643 | def run_purge(args: argparse.Namespace) -> None: |
| 644 | """Delete one or all local Muse index files under ``.muse/indices/``. |
| 645 | |
| 646 | Indexes are derived data — the canonical commit history and object store |
| 647 | are never modified. Any purged index can be fully recreated at any time |
| 648 | with ``muse code index rebuild``. |
| 649 | |
| 650 | Indexes that are not present are silently skipped and reported under |
| 651 | ``skipped`` rather than causing an error. |
| 652 | |
| 653 | Security: the ``--index`` argument is validated by argparse against |
| 654 | ``KNOWN_INDEX_NAMES`` before ``run_purge`` is called, and ``purge_index`` |
| 655 | performs a second validation before any ``unlink`` — no caller-supplied |
| 656 | path fragment can reach the file system. Only files under |
| 657 | ``.muse/indices/`` are ever deleted; the rest of the object store is |
| 658 | untouched. |
| 659 | |
| 660 | JSON output fields (``--json`` / ``-j``) |
| 661 | ----------------------------------------- |
| 662 | ``schema_version`` |
| 663 | Muse version string. |
| 664 | ``purged`` |
| 665 | List of index names whose files were found and deleted. |
| 666 | ``skipped`` |
| 667 | List of index names that were not present (nothing to delete). |
| 668 | |
| 669 | Exit codes |
| 670 | ---------- |
| 671 | 0 — operation completed (skipped indexes do NOT cause non-zero exit) |
| 672 | 2 — not inside a Muse repository |
| 673 | """ |
| 674 | index_name: str | None = args.index_name |
| 675 | as_json: bool = args.as_json |
| 676 | |
| 677 | root = require_repo() |
| 678 | muse_dir = root / ".muse" |
| 679 | names = list(KNOWN_INDEX_NAMES) if index_name is None else [index_name] |
| 680 | |
| 681 | purged: list[str] = [] |
| 682 | skipped: list[str] = [] |
| 683 | for name in names: |
| 684 | if purge_index(root, name): |
| 685 | purged.append(name) |
| 686 | else: |
| 687 | skipped.append(name) |
| 688 | |
| 689 | if as_json: |
| 690 | print(json.dumps({ |
| 691 | "schema_version": __version__, |
| 692 | "purged": purged, |
| 693 | "skipped": skipped, |
| 694 | })) |
| 695 | return |
| 696 | |
| 697 | for name in purged: |
| 698 | print(f" 🗑️ {name} — deleted") |
| 699 | for name in skipped: |
| 700 | print(f" ⬜ {name} — not present, nothing to delete") |
| 701 | if purged: |
| 702 | print(f"\nPurged {len(purged)} index(es). Run 'muse code index rebuild' to recreate.") |
File History
2 commits
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6
chore(tests): add docstring to tests/__init__.py so rc14 tr…
Human
32 days ago
sha256:d8316ffae901be06347e16ab55be11868eb519dd16ade3e8aa16a99e662f7e62
baseline: rc14 re-baseline after rc3 store corruption recovery
Human
patch
32 days ago