count_objects.py
python
sha256:ae1eca169c5efe00a6415971ed0e7259f68df3e3ce23935c4b1b714c2df6a240
Merge branch 'dev' into main
Human
22 days ago
| 1 | """``muse count-objects`` — object store diagnostics. |
| 2 | |
| 3 | Reports the number of loose objects in the local store, their total |
| 4 | on-disk size, and optional per-shard breakdown. A fast, read-only |
| 5 | diagnostic that uses ``os.scandir`` (stat only — no content reads) so |
| 6 | it stays sub-100 ms even for stores with 100k+ objects. |
| 7 | |
| 8 | Concepts |
| 9 | -------- |
| 10 | **Loose objects**: individual files stored under ``.muse/objects/<shard>/<id>``. |
| 11 | Muse does not yet have a pack-file format, so every object is loose. |
| 12 | |
| 13 | **Reachable objects**: objects referenced (directly or transitively) by |
| 14 | any branch tip. For each branch, the walk follows: commit → snapshot |
| 15 | manifest → blob object IDs. |
| 16 | |
| 17 | **Unreachable objects** (``--unreachable``): objects present in the store |
| 18 | but not reachable from any branch tip — candidates for ``muse prune``. |
| 19 | |
| 20 | JSON schema (``--json``):: |
| 21 | |
| 22 | { |
| 23 | "loose_objects": 1240, |
| 24 | "loose_size_kb": 4820.0, |
| 25 | "total_objects": 1240, |
| 26 | "total_size_kb": 4820.0, |
| 27 | "object_store_path": ".muse/objects", |
| 28 | "unreachable_objects": 0, # only when --unreachable is given |
| 29 | "shards": [ # only when --verbose is given |
| 30 | {"prefix": "ab", "count": 12, "size_kb": 48.0}, |
| 31 | ... |
| 32 | ], |
| 33 | "duration_ms": 0.003, |
| 34 | "exit_code": 0 |
| 35 | } |
| 36 | |
| 37 | ``duration_ms`` |
| 38 | Wall-clock time from argument parsing to output. |
| 39 | ``exit_code`` |
| 40 | Mirrors the process exit code: always ``0`` on success. |
| 41 | |
| 42 | Exit codes:: |
| 43 | |
| 44 | 0 — success |
| 45 | 2 — not a Muse repository |
| 46 | 3 — I/O error |
| 47 | |
| 48 | Examples:: |
| 49 | |
| 50 | muse count-objects |
| 51 | muse count-objects --json |
| 52 | muse count-objects --verbose |
| 53 | muse count-objects --unreachable |
| 54 | muse count-objects --unreachable --verbose --json |
| 55 | """ |
| 56 | |
| 57 | import argparse |
| 58 | import json as _json |
| 59 | import logging |
| 60 | import os |
| 61 | import pathlib |
| 62 | import sys |
| 63 | import time |
| 64 | from typing import TypedDict |
| 65 | |
| 66 | from muse.core.types import long_id, split_id |
| 67 | from muse.core.paths import objects_dir as _objects_dir |
| 68 | from muse.core.errors import ExitCode |
| 69 | from muse.core.graph import iter_ancestors |
| 70 | from muse.core.object_store import iter_stored_objects, objects_dir |
| 71 | from muse.core.refs import iter_branch_refs |
| 72 | from muse.core.repo import require_repo |
| 73 | from muse.core.envelope import EnvelopeJson, make_envelope |
| 74 | from muse.core.timing import start_timer |
| 75 | from muse.core.refs import get_head_commit_id |
| 76 | from muse.core.commits import read_commit |
| 77 | from muse.core.snapshots import read_snapshot |
| 78 | |
| 79 | logger = logging.getLogger(__name__) |
| 80 | |
| 81 | # --------------------------------------------------------------------------- |
| 82 | # TypedDicts for structured output |
| 83 | # --------------------------------------------------------------------------- |
| 84 | |
| 85 | class _ShardInfo(TypedDict): |
| 86 | prefix: str |
| 87 | count: int |
| 88 | size_kb: float |
| 89 | |
| 90 | class _CountObjectsJson(EnvelopeJson, total=False): |
| 91 | loose_objects: int |
| 92 | loose_size_kb: float |
| 93 | total_objects: int |
| 94 | total_size_kb: float |
| 95 | object_store_path: str |
| 96 | unreachable_objects: int # only present when --unreachable is given |
| 97 | shards: list[_ShardInfo] # only present when --verbose is given |
| 98 | |
| 99 | # --------------------------------------------------------------------------- |
| 100 | # Internal helpers |
| 101 | # --------------------------------------------------------------------------- |
| 102 | |
| 103 | def _count_loose_objects(root: pathlib.Path) -> tuple[int, int]: |
| 104 | """Count loose objects and their total on-disk size (in bytes). |
| 105 | |
| 106 | Uses ``os.scandir`` to stat each object file without reading content. |
| 107 | Traverses one level of shard directories under ``.muse/objects/``. |
| 108 | |
| 109 | Args: |
| 110 | root: Absolute repo root. |
| 111 | |
| 112 | Returns: |
| 113 | ``(count, total_bytes)`` tuple. |
| 114 | """ |
| 115 | total_count = 0 |
| 116 | total_bytes = 0 |
| 117 | |
| 118 | for _, obj_file in iter_stored_objects(root): |
| 119 | try: |
| 120 | total_bytes += obj_file.stat().st_size |
| 121 | total_count += 1 |
| 122 | except OSError: |
| 123 | pass |
| 124 | |
| 125 | return total_count, total_bytes |
| 126 | |
| 127 | def _count_by_shard(root: pathlib.Path) -> list[dict]: |
| 128 | """Return per-shard breakdown of object counts and sizes. |
| 129 | |
| 130 | Args: |
| 131 | root: Absolute repo root. |
| 132 | |
| 133 | Returns: |
| 134 | List of ``{"prefix": str, "count": int, "size_kb": float}`` dicts, |
| 135 | sorted by prefix. |
| 136 | """ |
| 137 | shard_stats: dict[str, dict] = {} |
| 138 | |
| 139 | for oid, obj_file in iter_stored_objects(root): |
| 140 | prefix = split_id(oid)[1][:2] |
| 141 | entry = shard_stats.setdefault(prefix, {"count": 0, "size_b": 0}) |
| 142 | try: |
| 143 | entry["size_b"] += obj_file.stat().st_size |
| 144 | entry["count"] += 1 |
| 145 | except OSError: |
| 146 | pass |
| 147 | |
| 148 | return [ |
| 149 | {"prefix": p, "count": v["count"], "size_kb": round(v["size_b"] / 1024, 2)} |
| 150 | for p, v in sorted(shard_stats.items()) |
| 151 | if v["count"] |
| 152 | ] |
| 153 | |
| 154 | def _collect_reachable_ids(root: pathlib.Path) -> set[str]: |
| 155 | """BFS walk from all branch tips to collect every referenced object ID. |
| 156 | |
| 157 | For each branch tip, walks: |
| 158 | - commit record → commit is not stored as a blob, so not counted |
| 159 | - snapshot manifest → each ``object_id`` value is a reachable blob |
| 160 | |
| 161 | Args: |
| 162 | root: Absolute repo root. |
| 163 | |
| 164 | Returns: |
| 165 | Set of object ID strings (SHA-256 hex) that are reachable from |
| 166 | any branch tip. |
| 167 | """ |
| 168 | reachable: set[str] = set() |
| 169 | |
| 170 | # Collect all branch tips. |
| 171 | tips: list[str] = [cid for _, cid in iter_branch_refs(root)] |
| 172 | if not tips: |
| 173 | return reachable |
| 174 | |
| 175 | # BFS from all tips; collect commit, snapshot, and blob IDs. |
| 176 | for commit in iter_ancestors(root, tips, first_parent_only=True): |
| 177 | reachable.add(commit.commit_id) |
| 178 | reachable.add(commit.snapshot_id) |
| 179 | snap = read_snapshot(root, commit.snapshot_id) |
| 180 | if snap is not None: |
| 181 | reachable.update(snap.manifest.values()) |
| 182 | |
| 183 | return reachable |
| 184 | |
| 185 | # --------------------------------------------------------------------------- |
| 186 | # Registration |
| 187 | # --------------------------------------------------------------------------- |
| 188 | |
| 189 | def register( |
| 190 | subparsers: "argparse._SubParsersAction[argparse.ArgumentParser]", |
| 191 | ) -> None: |
| 192 | """Register the ``muse count-objects`` subcommand.""" |
| 193 | parser = subparsers.add_parser( |
| 194 | "count-objects", |
| 195 | help="Count objects in the local store and report store size.", |
| 196 | description=__doc__, |
| 197 | formatter_class=argparse.RawDescriptionHelpFormatter, |
| 198 | ) |
| 199 | parser.add_argument( |
| 200 | "-v", "--verbose", |
| 201 | action="store_true", |
| 202 | dest="verbose", |
| 203 | help="Show per-shard object counts and sizes.", |
| 204 | ) |
| 205 | parser.add_argument( |
| 206 | "--unreachable", |
| 207 | action="store_true", |
| 208 | dest="unreachable", |
| 209 | help=( |
| 210 | "Also count objects not reachable from any branch tip — " |
| 211 | "these are candidates for 'muse prune'." |
| 212 | ), |
| 213 | ) |
| 214 | parser.add_argument( |
| 215 | "--json", "-j", |
| 216 | action="store_true", |
| 217 | dest="json_out", |
| 218 | help="Emit machine-readable JSON on stdout.", |
| 219 | ) |
| 220 | parser.set_defaults(func=run) |
| 221 | |
| 222 | # --------------------------------------------------------------------------- |
| 223 | # Run |
| 224 | # --------------------------------------------------------------------------- |
| 225 | |
| 226 | def run(args: argparse.Namespace) -> None: |
| 227 | """Count objects in the local store. |
| 228 | |
| 229 | Uses stat-only traversal of the object store — no object content is read. |
| 230 | Use ``--verbose`` to see a per-shard breakdown; ``--unreachable`` to count |
| 231 | objects not referenced by any commit. |
| 232 | |
| 233 | Agent quickstart |
| 234 | ---------------- |
| 235 | :: |
| 236 | |
| 237 | muse count-objects --json |
| 238 | muse count-objects --verbose --json |
| 239 | muse count-objects --unreachable --json |
| 240 | |
| 241 | JSON fields |
| 242 | ----------- |
| 243 | loose_objects Number of loose object files. |
| 244 | loose_size_kb Total size of loose objects in KiB. |
| 245 | total_objects Total object count. |
| 246 | total_size_kb Total size in KiB. |
| 247 | object_store_path Absolute path to the object store. |
| 248 | unreachable_objects Count of unreachable objects (with ``--unreachable``). |
| 249 | shards Per-shard breakdown list (with ``--verbose``). |
| 250 | |
| 251 | Exit codes |
| 252 | ---------- |
| 253 | 0 Success. |
| 254 | 2 Not inside a Muse repository. |
| 255 | 3 I/O error. |
| 256 | """ |
| 257 | elapsed = start_timer() |
| 258 | verbose: bool = args.verbose |
| 259 | check_unreachable: bool = args.unreachable |
| 260 | json_out: bool = args.json_out |
| 261 | |
| 262 | root = require_repo() |
| 263 | |
| 264 | try: |
| 265 | loose_count, loose_bytes = _count_loose_objects(root) |
| 266 | except OSError as exc: |
| 267 | print(f"❌ I/O error reading object store: {exc}", file=sys.stderr) |
| 268 | raise SystemExit(ExitCode.IO_ERROR) |
| 269 | |
| 270 | loose_kb = round(loose_bytes / 1024, 2) |
| 271 | store_path = str(_objects_dir(root).relative_to(root)) |
| 272 | |
| 273 | unreachable_count: int | None = None |
| 274 | if check_unreachable: |
| 275 | reachable = _collect_reachable_ids(root) |
| 276 | unreachable_count = sum( |
| 277 | 1 for oid, _ in iter_stored_objects(root) if oid not in reachable |
| 278 | ) |
| 279 | |
| 280 | shards: list[_ShardInfo] | None = None |
| 281 | if verbose: |
| 282 | shards = _count_by_shard(root) |
| 283 | |
| 284 | if json_out: |
| 285 | out = _CountObjectsJson( |
| 286 | **make_envelope(elapsed), |
| 287 | loose_objects=loose_count, |
| 288 | loose_size_kb=loose_kb, |
| 289 | total_objects=loose_count, |
| 290 | total_size_kb=loose_kb, |
| 291 | object_store_path=store_path, |
| 292 | ) |
| 293 | if unreachable_count is not None: |
| 294 | out["unreachable_objects"] = unreachable_count |
| 295 | if shards is not None: |
| 296 | out["shards"] = shards |
| 297 | print(_json.dumps(out)) |
| 298 | return |
| 299 | |
| 300 | # ── Text output ──────────────────────────────────────────────────────── |
| 301 | print(f"loose objects: {loose_count}") |
| 302 | print(f"loose size: {loose_kb:.1f} KiB") |
| 303 | if unreachable_count is not None: |
| 304 | print(f"unreachable: {unreachable_count}") |
| 305 | if shards is not None: |
| 306 | for shard in shards: |
| 307 | print(f" shard {shard['prefix']}: {shard['count']} objects, {shard['size_kb']:.1f} KiB") |
File History
3 commits
sha256:ae1eca169c5efe00a6415971ed0e7259f68df3e3ce23935c4b1b714c2df6a240
Merge branch 'dev' into main
Human
22 days ago
sha256:50d413a635f57761c3fce1f70251b957b6423821525bf330747ef4007339222e
Merge branch 'dev' into main
Human
33 days ago
sha256:fb67fed5a4d3e40de84bdd163de94ef1386570bef1dd1a020a732c8a038962ce
Merge branch 'dev' into main
Human
52 days ago