clones.py
file-level
1
files
1
commits
0
hotspots
0
🧊 dead
0
💥 blast risk
| 1 | """muse code clones — find duplicate and near-duplicate symbols. |
| 2 | |
| 3 | Detects two tiers of code duplication from committed snapshot data: |
| 4 | |
| 5 | **Exact clones** |
| 6 | Symbols with the same ``body_hash`` at different addresses. The body is |
| 7 | character-for-character identical (after normalisation) even if the name or |
| 8 | surrounding context differs. These are true copy-paste duplicates. |
| 9 | |
| 10 | **Near-clones** |
| 11 | Symbols with the same ``signature_id`` but different ``body_hash``. Same |
| 12 | function signature, different implementation — strong candidates for |
| 13 | consolidation behind a shared abstraction. |
| 14 | |
| 15 | Git has no concept of these. Git stores file diffs; Muse stores symbol |
| 16 | identity hashes. Clone detection is a single pass over the snapshot index. |
| 17 | |
| 18 | Usage:: |
| 19 | |
| 20 | muse code clones |
| 21 | muse code clones --tier exact |
| 22 | muse code clones --tier near |
| 23 | muse code clones --kind function |
| 24 | muse code clones --language Python |
| 25 | muse code clones --file muse/core/ |
| 26 | muse code clones --exclude-same-file |
| 27 | muse code clones --commit HEAD~10 |
| 28 | muse code clones --min-cluster 3 |
| 29 | muse code clones --json |
| 30 | |
| 31 | Output:: |
| 32 | |
| 33 | Clone analysis — commit a1b2c3d4 |
| 34 | |
| 35 | Exact clones (2 clusters): |
| 36 | body_hash a1b2c3d4: |
| 37 | src/billing.py::compute_hash function |
| 38 | src/utils.py::compute_hash function |
| 39 | src/legacy.py::_hash function |
| 40 | |
| 41 | Near-clones — same signature (3 clusters): |
| 42 | signature_id e5f6a7b8: |
| 43 | src/billing.py::validate function |
| 44 | src/auth.py::validate function |
| 45 | |
| 46 | Files with most clone members: |
| 47 | src/billing.py 4 clone symbols |
| 48 | |
| 49 | Flags: |
| 50 | |
| 51 | ``--tier {exact|near|both}`` |
| 52 | Which tier to report (default: both). |
| 53 | |
| 54 | ``--kind KIND`` |
| 55 | Restrict to symbols of this kind (function, class, method, …). |
| 56 | |
| 57 | ``--language LANG`` |
| 58 | Restrict to files of this language. |
| 59 | |
| 60 | ``--file PATH`` |
| 61 | Restrict to symbols whose file path starts with PATH. |
| 62 | |
| 63 | ``--exclude-same-file`` |
| 64 | Skip clusters where every member lives in the same file. |
| 65 | |
| 66 | ``--min-cluster N`` |
| 67 | Only show clusters with at least N members (default: 2). |
| 68 | |
| 69 | ``--commit, -c REF`` |
| 70 | Analyse a historical snapshot instead of HEAD. |
| 71 | |
| 72 | ``--json`` |
| 73 | Emit results as JSON. |
| 74 | """ |
| 75 | |
| 76 | from __future__ import annotations |
| 77 | |
| 78 | import argparse |
| 79 | import json |
| 80 | import logging |
| 81 | import pathlib |
| 82 | from typing import Literal, TypedDict |
| 83 | |
| 84 | from muse._version import __version__ |
| 85 | from muse.core.errors import ExitCode |
| 86 | from muse.core.repo import read_repo_id, require_repo |
| 87 | from muse.core._types import Manifest |
| 88 | from muse.core.store import get_commit_snapshot_manifest, read_current_branch, resolve_commit_ref |
| 89 | from muse.core.symbol_cache import SymbolCache, load_symbol_cache |
| 90 | from muse.plugins.code._query import language_of, symbols_for_snapshot |
| 91 | from muse.plugins.code.ast_parser import SymbolRecord |
| 92 | from muse.core.validation import clamp_int, sanitize_display |
| 93 | |
| 94 | type _SymIndex = dict[str, list[tuple[str, SymbolRecord]]] |
| 95 | type _FileCountMap = dict[str, int] |
| 96 | |
| 97 | logger = logging.getLogger(__name__) |
| 98 | |
| 99 | CloneTier = Literal["exact", "near", "both"] |
| 100 | |
| 101 | |
| 102 | # --------------------------------------------------------------------------- |
| 103 | # Typed output shapes |
| 104 | # --------------------------------------------------------------------------- |
| 105 | |
| 106 | |
| 107 | class _MemberDict(TypedDict): |
| 108 | address: str |
| 109 | kind: str |
| 110 | language: str |
| 111 | body_hash: str |
| 112 | signature_id: str |
| 113 | content_id: str |
| 114 | |
| 115 | |
| 116 | class _ClusterDict(TypedDict): |
| 117 | tier: str |
| 118 | hash: str |
| 119 | count: int |
| 120 | members: list[_MemberDict] |
| 121 | |
| 122 | |
| 123 | class _FileHotspot(TypedDict): |
| 124 | file: str |
| 125 | clone_symbols: int |
| 126 | |
| 127 | |
| 128 | # --------------------------------------------------------------------------- |
| 129 | # Core data model |
| 130 | # --------------------------------------------------------------------------- |
| 131 | |
| 132 | |
| 133 | class _CloneCluster: |
| 134 | """A group of symbols that are duplicates of each other.""" |
| 135 | |
| 136 | def __init__( |
| 137 | self, |
| 138 | tier: CloneTier, |
| 139 | hash_value: str, |
| 140 | members: list[tuple[str, SymbolRecord]], |
| 141 | ) -> None: |
| 142 | self.tier = tier |
| 143 | self.hash_value = hash_value |
| 144 | self.members = members # (address, record) |
| 145 | |
| 146 | def to_dict(self) -> _ClusterDict: |
| 147 | return { |
| 148 | "tier": self.tier, |
| 149 | "hash": self.hash_value[:8], |
| 150 | "count": len(self.members), # int — not str |
| 151 | "members": [ |
| 152 | _MemberDict( |
| 153 | address=addr, |
| 154 | kind=rec["kind"], |
| 155 | language=language_of(addr.split("::")[0]), |
| 156 | body_hash=rec["body_hash"][:8], |
| 157 | signature_id=rec["signature_id"][:8], |
| 158 | content_id=rec["content_id"][:8], |
| 159 | ) |
| 160 | for addr, rec in self.members |
| 161 | ], |
| 162 | } |
| 163 | |
| 164 | |
| 165 | # --------------------------------------------------------------------------- |
| 166 | # Detection logic |
| 167 | # --------------------------------------------------------------------------- |
| 168 | |
| 169 | |
| 170 | def find_clones( |
| 171 | root: pathlib.Path, |
| 172 | manifest: Manifest, |
| 173 | tier: CloneTier, |
| 174 | kind_filter: str | None, |
| 175 | min_cluster: int, |
| 176 | *, |
| 177 | language_filter: str | None = None, |
| 178 | file_filter: str | None = None, |
| 179 | exclude_same_file: bool = False, |
| 180 | cache: SymbolCache | None = None, |
| 181 | ) -> list[_CloneCluster]: |
| 182 | """Build clone clusters from *manifest*. |
| 183 | |
| 184 | Args: |
| 185 | root: Repository root (object store location). |
| 186 | manifest: Snapshot manifest: file path → SHA-256 object ID. |
| 187 | tier: Which clone tier(s) to detect: "exact", "near", "both". |
| 188 | kind_filter: If set, only analyse symbols of this kind. |
| 189 | min_cluster: Minimum cluster size to report (default 2). |
| 190 | language_filter: If set, only analyse files of this language. |
| 191 | file_filter: If set, only analyse symbols whose file path starts |
| 192 | with this prefix (e.g. "muse/core/"). |
| 193 | exclude_same_file: If True, skip clusters where all members are in |
| 194 | the same file (eliminates test-helper noise). |
| 195 | cache: Optional shared ``SymbolCache``. Pass one to avoid |
| 196 | re-parsing blobs when the caller has a warm cache. |
| 197 | """ |
| 198 | sym_map = symbols_for_snapshot( |
| 199 | root, |
| 200 | manifest, |
| 201 | kind_filter=kind_filter, |
| 202 | language_filter=language_filter, |
| 203 | cache=cache, |
| 204 | ) |
| 205 | |
| 206 | # Flatten to list of (address, record), applying file_filter. |
| 207 | all_syms: list[tuple[str, SymbolRecord]] = [ |
| 208 | (addr, rec) |
| 209 | for fp, tree in sorted(sym_map.items()) |
| 210 | if file_filter is None or fp.startswith(file_filter) |
| 211 | for addr, rec in sorted(tree.items()) |
| 212 | if rec["kind"] != "import" |
| 213 | ] |
| 214 | |
| 215 | clusters: list[_CloneCluster] = [] |
| 216 | |
| 217 | if tier in ("exact", "both"): |
| 218 | body_index: _SymIndex = {} |
| 219 | for addr, rec in all_syms: |
| 220 | body_index.setdefault(rec["body_hash"], []).append((addr, rec)) |
| 221 | for body_hash, members in sorted(body_index.items()): |
| 222 | if len(members) < min_cluster: |
| 223 | continue |
| 224 | if exclude_same_file and _all_same_file(members): |
| 225 | continue |
| 226 | clusters.append(_CloneCluster("exact", body_hash, members)) |
| 227 | |
| 228 | if tier in ("near", "both"): |
| 229 | sig_index: _SymIndex = {} |
| 230 | for addr, rec in all_syms: |
| 231 | sig_index.setdefault(rec["signature_id"], []).append((addr, rec)) |
| 232 | for sig_id, members in sorted(sig_index.items()): |
| 233 | # Near-clone: same signature, at least two DIFFERENT body hashes. |
| 234 | unique_bodies = {r["body_hash"] for _, r in members} |
| 235 | if len(members) < min_cluster or len(unique_bodies) <= 1: |
| 236 | continue |
| 237 | if exclude_same_file and _all_same_file(members): |
| 238 | continue |
| 239 | clusters.append(_CloneCluster("near", sig_id, members)) |
| 240 | |
| 241 | # Sort: largest clusters first, then by tier (exact before near), then hash. |
| 242 | clusters.sort(key=lambda c: (-len(c.members), c.tier, c.hash_value)) |
| 243 | return clusters |
| 244 | |
| 245 | |
| 246 | def _all_same_file(members: list[tuple[str, SymbolRecord]]) -> bool: |
| 247 | """Return True when every member lives in the same source file.""" |
| 248 | files = {addr.split("::")[0] for addr, _ in members} |
| 249 | return len(files) == 1 |
| 250 | |
| 251 | |
| 252 | def _file_hotspots( |
| 253 | clusters: list[_CloneCluster], |
| 254 | top: int = 10, |
| 255 | ) -> list[_FileHotspot]: |
| 256 | """Rank files by the number of clone-member symbols they contain.""" |
| 257 | file_counts: _FileCountMap = {} |
| 258 | for cluster in clusters: |
| 259 | for addr, _ in cluster.members: |
| 260 | fp = addr.split("::")[0] |
| 261 | file_counts[fp] = file_counts.get(fp, 0) + 1 |
| 262 | ranked = sorted(file_counts.items(), key=lambda t: t[1], reverse=True)[:top] |
| 263 | return [_FileHotspot(file=fp, clone_symbols=cnt) for fp, cnt in ranked] |
| 264 | |
| 265 | |
| 266 | # --------------------------------------------------------------------------- |
| 267 | # CLI registration and entry point |
| 268 | # --------------------------------------------------------------------------- |
| 269 | |
| 270 | |
| 271 | def register(subparsers: "argparse._SubParsersAction[argparse.ArgumentParser]") -> None: |
| 272 | """Register the clones subcommand.""" |
| 273 | parser = subparsers.add_parser( |
| 274 | "clones", |
| 275 | help="Find exact and near-duplicate symbols in the committed snapshot.", |
| 276 | description=__doc__, |
| 277 | formatter_class=argparse.RawDescriptionHelpFormatter, |
| 278 | ) |
| 279 | parser.add_argument( |
| 280 | "--tier", "-t", default="both", choices=["exact", "near", "both"], |
| 281 | help="Tier to report: exact, near, or both (default: both).", |
| 282 | ) |
| 283 | parser.add_argument( |
| 284 | "--kind", "-k", default=None, metavar="KIND", dest="kind_filter", |
| 285 | help="Restrict to symbols of this kind (function, class, method, …).", |
| 286 | ) |
| 287 | parser.add_argument( |
| 288 | "--language", "-l", default=None, metavar="LANG", dest="language_filter", |
| 289 | help="Restrict to files of this language.", |
| 290 | ) |
| 291 | parser.add_argument( |
| 292 | "--file", "-f", default=None, metavar="PATH", dest="file_filter", |
| 293 | help="Restrict to symbols whose file path starts with PATH.", |
| 294 | ) |
| 295 | parser.add_argument( |
| 296 | "--exclude-same-file", action="store_true", dest="exclude_same_file", |
| 297 | help="Skip clusters where all members are in the same file.", |
| 298 | ) |
| 299 | parser.add_argument( |
| 300 | "--min-cluster", "-m", type=int, default=2, metavar="N", dest="min_cluster", |
| 301 | help="Only show clusters with at least N members (default: 2).", |
| 302 | ) |
| 303 | parser.add_argument( |
| 304 | "--commit", "-c", default=None, metavar="REF", dest="ref", |
| 305 | help="Analyse this commit instead of HEAD.", |
| 306 | ) |
| 307 | parser.add_argument( |
| 308 | "--json", action="store_true", dest="as_json", |
| 309 | help="Emit results as JSON.", |
| 310 | ) |
| 311 | parser.set_defaults(func=run) |
| 312 | |
| 313 | |
| 314 | def run(args: argparse.Namespace) -> None: |
| 315 | """Find exact and near-duplicate symbols in the committed snapshot. |
| 316 | |
| 317 | Exact clones share the same ``body_hash`` (identical implementation). |
| 318 | Near-clones share the same ``signature_id`` but differ in body — same |
| 319 | contract, different implementation. Both are candidates for consolidation |
| 320 | behind shared abstractions. |
| 321 | |
| 322 | Uses content-addressed hashes from the snapshot index — no AST |
| 323 | recomputation at query time. A shared SymbolCache ensures each blob is |
| 324 | parsed at most once across all analysis passes. |
| 325 | """ |
| 326 | tier: CloneTier = args.tier |
| 327 | kind_filter: str | None = args.kind_filter |
| 328 | language_filter: str | None = args.language_filter |
| 329 | file_filter: str | None = args.file_filter |
| 330 | exclude_same_file: bool = args.exclude_same_file |
| 331 | min_cluster: int = clamp_int(args.min_cluster, 1, 10000, 'min_cluster') |
| 332 | ref: str | None = args.ref |
| 333 | as_json: bool = args.as_json |
| 334 | |
| 335 | if min_cluster < 2: |
| 336 | logger.error("--min-cluster must be at least 2, got %d", min_cluster) |
| 337 | raise SystemExit(ExitCode.USER_ERROR) |
| 338 | |
| 339 | root = require_repo() |
| 340 | |
| 341 | from muse.plugins.registry import read_domain |
| 342 | try: |
| 343 | if read_domain(root) == "knowtation": |
| 344 | from muse.plugins.knowtation.cli_hooks import run_clones_for_vault |
| 345 | run_clones_for_vault(root, args) |
| 346 | return |
| 347 | except Exception: # noqa: BLE001 |
| 348 | pass |
| 349 | |
| 350 | try: |
| 351 | repo_id = read_repo_id(root) |
| 352 | except (FileNotFoundError, json.JSONDecodeError, KeyError) as exc: |
| 353 | logger.error("Cannot read repo identity: %s", exc) |
| 354 | raise SystemExit(ExitCode.INTERNAL_ERROR) from exc |
| 355 | |
| 356 | branch = read_current_branch(root) |
| 357 | commit = resolve_commit_ref(root, repo_id, branch, ref) |
| 358 | if commit is None: |
| 359 | logger.error("Commit %r not found.", ref or "HEAD") |
| 360 | raise SystemExit(ExitCode.USER_ERROR) |
| 361 | |
| 362 | manifest = get_commit_snapshot_manifest(root, commit.commit_id) or {} |
| 363 | |
| 364 | # Shared cache — each blob parsed at most once. |
| 365 | cache = load_symbol_cache(root) |
| 366 | |
| 367 | cluster_list = find_clones( |
| 368 | root, |
| 369 | manifest, |
| 370 | tier, |
| 371 | kind_filter, |
| 372 | min_cluster, |
| 373 | language_filter=language_filter, |
| 374 | file_filter=file_filter, |
| 375 | exclude_same_file=exclude_same_file, |
| 376 | cache=cache, |
| 377 | ) |
| 378 | |
| 379 | exact_clusters = [c for c in cluster_list if c.tier == "exact"] |
| 380 | near_clusters = [c for c in cluster_list if c.tier == "near"] |
| 381 | total_symbols = sum(len(c.members) for c in cluster_list) |
| 382 | hotspots = _file_hotspots(cluster_list) |
| 383 | |
| 384 | if as_json: |
| 385 | print(json.dumps( |
| 386 | { |
| 387 | "schema_version": __version__, |
| 388 | "commit": commit.commit_id[:8], |
| 389 | "branch": branch, |
| 390 | "tier": tier, |
| 391 | "min_cluster": min_cluster, |
| 392 | "kind_filter": kind_filter, |
| 393 | "language_filter": language_filter, |
| 394 | "file_filter": file_filter, |
| 395 | "exclude_same_file": exclude_same_file, |
| 396 | "exact_clone_clusters": len(exact_clusters), |
| 397 | "near_clone_clusters": len(near_clusters), |
| 398 | "total_symbols_involved": total_symbols, |
| 399 | "file_hotspots": hotspots, |
| 400 | "clusters": [c.to_dict() for c in cluster_list], |
| 401 | }, |
| 402 | indent=2, |
| 403 | )) |
| 404 | return |
| 405 | |
| 406 | print(f"\nClone analysis — commit {commit.commit_id[:8]}") |
| 407 | if kind_filter: |
| 408 | print(f" (kind: {kind_filter})") |
| 409 | if language_filter: |
| 410 | print(f" (language: {language_filter})") |
| 411 | if file_filter: |
| 412 | print(f" (file prefix: {file_filter})") |
| 413 | if exclude_same_file: |
| 414 | print(" (same-file clusters excluded)") |
| 415 | print("─" * 62) |
| 416 | |
| 417 | if not cluster_list: |
| 418 | print("\n ✅ No clones detected.") |
| 419 | return |
| 420 | |
| 421 | if exact_clusters and tier in ("exact", "both"): |
| 422 | print(f"\nExact clones ({len(exact_clusters)} cluster(s)):") |
| 423 | for cl in exact_clusters: |
| 424 | print(f" body_hash {cl.hash_value[:8]}:") |
| 425 | for addr, rec in cl.members: |
| 426 | print(f" {sanitize_display(addr)} {rec['kind']}") |
| 427 | |
| 428 | if near_clusters and tier in ("near", "both"): |
| 429 | print(f"\nNear-clones — same signature ({len(near_clusters)} cluster(s)):") |
| 430 | for cl in near_clusters: |
| 431 | print(f" signature_id {cl.hash_value[:8]}:") |
| 432 | for addr, rec in cl.members: |
| 433 | print(f" {sanitize_display(addr)} {rec['kind']} (body {rec['body_hash'][:8]})") |
| 434 | |
| 435 | print(f"\n {len(cluster_list)} clone cluster(s), {total_symbols} total symbol(s) involved") |
| 436 | |
| 437 | if hotspots: |
| 438 | print(f"\nFiles with most clone members:") |
| 439 | for h in hotspots[:5]: |
| 440 | print(f" {sanitize_display(h['file'])} {h['clone_symbols']} clone symbol(s)") |