note_metrics.py
python
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6
chore(tests): add docstring to tests/__init__.py so rc14 tr…
Human
26 days ago
| 1 | """Knowtation note metrics — Phase 3.3 helpers. |
| 2 | |
| 3 | This module supplies the vault-aware counterparts to six ``muse code`` commands |
| 4 | that the CLI routes to when the active domain is ``knowtation``: |
| 5 | |
| 6 | ============= ====================================================== |
| 7 | CLI command Helper exported here |
| 8 | ============= ====================================================== |
| 9 | hotspots :func:`note_hotspots` — note churn by commit count |
| 10 | gravity :func:`note_gravity` — fraction of vault linking in |
| 11 | dead :func:`note_dead` — notes with zero inbound links |
| 12 | clones :func:`note_clones` — duplicate / near-duplicate notes |
| 13 | entangle :func:`note_entangle` — co-change note pairs |
| 14 | velocity :func:`note_velocity` — note additions per commit window |
| 15 | ============= ====================================================== |
| 16 | |
| 17 | Each helper returns a plain Python dict suitable for ``json.dumps``. |
| 18 | |
| 19 | Design |
| 20 | ------ |
| 21 | |
| 22 | - Helpers that need a commit graph accept ``commits`` (a sequence of |
| 23 | :class:`~muse.core.store.CommitRecord`) and use ``snapshot_manifest`` lookups |
| 24 | per commit; we never re-walk the DAG inside helpers. |
| 25 | - Helpers that need the live link graph accept a pre-built |
| 26 | :class:`~muse.plugins.knowtation.link_index.LinkIndex`. The CLI builds it |
| 27 | once and passes it through. |
| 28 | - All helpers exclude templates / meta / hidden directories implicitly, |
| 29 | because the link index already does this and the manifest-walk helpers |
| 30 | delegate to :func:`is_note`. |
| 31 | |
| 32 | Performance budgets (per Plan §3.3): |
| 33 | - < 2s on a 5k-note vault for each command. |
| 34 | """ |
| 35 | |
| 36 | from __future__ import annotations |
| 37 | |
| 38 | import logging |
| 39 | import pathlib |
| 40 | from collections import Counter, defaultdict |
| 41 | from typing import Any, Iterable |
| 42 | |
| 43 | from muse.plugins.knowtation._query import is_note |
| 44 | from muse.plugins.knowtation.link_index import LinkIndex |
| 45 | |
| 46 | logger = logging.getLogger(__name__) |
| 47 | |
| 48 | |
| 49 | # --------------------------------------------------------------------------- |
| 50 | # Manifest helpers |
| 51 | # --------------------------------------------------------------------------- |
| 52 | |
| 53 | |
| 54 | def _notes_in_manifest(manifest: dict[str, str]) -> dict[str, str]: |
| 55 | return {p: h for p, h in manifest.items() if is_note(p)} |
| 56 | |
| 57 | |
| 58 | def _changed_notes_between( |
| 59 | older: dict[str, str], newer: dict[str, str] |
| 60 | ) -> set[str]: |
| 61 | """Set of note paths whose content_hash differs (added, removed, or modified). |
| 62 | |
| 63 | Args: |
| 64 | older: Older snapshot manifest. |
| 65 | newer: Newer snapshot manifest. |
| 66 | |
| 67 | Returns: |
| 68 | Set of note paths that changed. |
| 69 | """ |
| 70 | changed: set[str] = set() |
| 71 | older_notes = _notes_in_manifest(older) |
| 72 | newer_notes = _notes_in_manifest(newer) |
| 73 | |
| 74 | for path, new_hash in newer_notes.items(): |
| 75 | if older_notes.get(path) != new_hash: |
| 76 | changed.add(path) |
| 77 | for path in older_notes: |
| 78 | if path not in newer_notes: |
| 79 | changed.add(path) |
| 80 | return changed |
| 81 | |
| 82 | |
| 83 | # --------------------------------------------------------------------------- |
| 84 | # hotspots — note churn leaderboard |
| 85 | # --------------------------------------------------------------------------- |
| 86 | |
| 87 | |
| 88 | def note_hotspots( |
| 89 | commits: list, |
| 90 | snapshot_loader: Any, |
| 91 | *, |
| 92 | top: int = 20, |
| 93 | min_changes: int = 1, |
| 94 | project_filter: str | None = None, |
| 95 | ) -> dict[str, Any]: |
| 96 | """Rank notes by the number of commits in *commits* that touched them. |
| 97 | |
| 98 | Args: |
| 99 | commits: List of CommitRecord, ordered newest-first. |
| 100 | snapshot_loader: Callable ``(commit_id) → manifest`` for fetching |
| 101 | each commit's snapshot manifest. Pass a closure |
| 102 | that delegates to ``get_commit_snapshot_manifest``. |
| 103 | top: Maximum entries in the returned ``ranking`` list. |
| 104 | min_changes: Minimum change count required for inclusion. |
| 105 | project_filter: Optional path-prefix filter (e.g. ``"projects/"``). |
| 106 | |
| 107 | Returns: |
| 108 | Dict with keys: |
| 109 | |
| 110 | - ``commits_analysed``: int — number of commits considered. |
| 111 | - ``total_notes_touched``: int — distinct notes that changed at least |
| 112 | once. |
| 113 | - ``ranking``: list of ``{path, changes}`` dicts, sorted by |
| 114 | ``changes`` (descending) then path (ascending). |
| 115 | """ |
| 116 | counts: Counter[str] = Counter() |
| 117 | prev_manifest: dict[str, str] | None = None |
| 118 | |
| 119 | # Walk newest → oldest; compare each commit with its predecessor (older). |
| 120 | for commit in commits: |
| 121 | try: |
| 122 | current = snapshot_loader(commit.commit_id) or {} |
| 123 | except Exception as exc: |
| 124 | logger.debug("hotspots: snapshot load failed for %s: %s", commit.commit_id, exc) |
| 125 | continue |
| 126 | if prev_manifest is None: |
| 127 | prev_manifest = current |
| 128 | continue |
| 129 | # commit is older than prev_manifest commit |
| 130 | changed = _changed_notes_between(current, prev_manifest) |
| 131 | for path in changed: |
| 132 | if project_filter and not path.startswith(project_filter): |
| 133 | continue |
| 134 | counts[path] += 1 |
| 135 | prev_manifest = current |
| 136 | |
| 137 | ranking_pairs = [(path, n) for path, n in counts.items() if n >= min_changes] |
| 138 | ranking_pairs.sort(key=lambda x: (-x[1], x[0])) |
| 139 | top_n = ranking_pairs[:top] |
| 140 | |
| 141 | return { |
| 142 | "commits_analysed": len(commits), |
| 143 | "total_notes_touched": len(counts), |
| 144 | "ranking": [{"path": path, "changes": n} for path, n in top_n], |
| 145 | } |
| 146 | |
| 147 | |
| 148 | # --------------------------------------------------------------------------- |
| 149 | # gravity — structural weight (fraction of vault linking to a note) |
| 150 | # --------------------------------------------------------------------------- |
| 151 | |
| 152 | |
| 153 | def note_gravity( |
| 154 | index: LinkIndex, |
| 155 | *, |
| 156 | top: int = 20, |
| 157 | min_inbound: int = 1, |
| 158 | ) -> dict[str, Any]: |
| 159 | """Rank notes by structural weight (inbound link count) in the live graph. |
| 160 | |
| 161 | Gravity = ``incoming_count(note) / total_notes`` — the fraction of the |
| 162 | vault that depends on this note. |
| 163 | |
| 164 | Args: |
| 165 | index: Pre-built :class:`LinkIndex`. |
| 166 | top: Maximum entries in ``ranking``. |
| 167 | min_inbound: Minimum inbound link count for inclusion. |
| 168 | |
| 169 | Returns: |
| 170 | Dict with keys ``total_notes``, ``ranking`` (list of |
| 171 | ``{path, inbound, gravity}`` dicts sorted by ``inbound`` descending). |
| 172 | """ |
| 173 | total_notes = max(1, index.notes_indexed) |
| 174 | counts: dict[str, int] = {} |
| 175 | for target, links in index.backward.items(): |
| 176 | # Count distinct source notes (a single source may link multiple times). |
| 177 | counts[target] = len({link.source for link in links}) |
| 178 | |
| 179 | pairs = [(p, n) for p, n in counts.items() if n >= min_inbound] |
| 180 | pairs.sort(key=lambda x: (-x[1], x[0])) |
| 181 | top_n = pairs[:top] |
| 182 | |
| 183 | return { |
| 184 | "total_notes": total_notes, |
| 185 | "ranking": [ |
| 186 | { |
| 187 | "path": path, |
| 188 | "inbound": n, |
| 189 | "gravity": round(n / total_notes, 6), |
| 190 | } |
| 191 | for path, n in top_n |
| 192 | ], |
| 193 | } |
| 194 | |
| 195 | |
| 196 | # --------------------------------------------------------------------------- |
| 197 | # dead — notes with zero inbound links and old mtime |
| 198 | # --------------------------------------------------------------------------- |
| 199 | |
| 200 | |
| 201 | def note_dead( |
| 202 | index: LinkIndex, |
| 203 | *, |
| 204 | root: pathlib.Path, |
| 205 | use_mtime: bool = True, |
| 206 | mtime_days_threshold: int = 180, |
| 207 | ) -> dict[str, Any]: |
| 208 | """Find notes with no inbound links (and optionally, old mtime). |
| 209 | |
| 210 | Per Plan §3.3: until Phase 4 lands the agent-memory-event signal, dead |
| 211 | detection uses inbound links + mtime, gated behind ``use_mtime`` to keep |
| 212 | the strict definition (no inbound only) available too. |
| 213 | |
| 214 | Args: |
| 215 | index: Pre-built :class:`LinkIndex`. |
| 216 | root: Vault root (used for mtime lookups when |
| 217 | ``use_mtime=True``). |
| 218 | use_mtime: When ``True``, also require mtime older than |
| 219 | ``mtime_days_threshold`` days. |
| 220 | mtime_days_threshold: Mtime cutoff in days. |
| 221 | |
| 222 | Returns: |
| 223 | Dict with keys: |
| 224 | |
| 225 | - ``total_notes_indexed``: int |
| 226 | - ``dead_count``: int |
| 227 | - ``dead_notes``: sorted list of ``{path, mtime?}`` dicts. |
| 228 | """ |
| 229 | import time |
| 230 | |
| 231 | cutoff = time.time() - (mtime_days_threshold * 86400) |
| 232 | dead_notes: list[dict[str, Any]] = [] |
| 233 | |
| 234 | for path in sorted(index.forward.keys()): |
| 235 | # No inbound resolved links means dead-by-link definition. |
| 236 | if path in index.backward and index.backward[path]: |
| 237 | continue |
| 238 | |
| 239 | entry: dict[str, Any] = {"path": path} |
| 240 | if use_mtime: |
| 241 | try: |
| 242 | mtime = (root / path).stat().st_mtime |
| 243 | except OSError: |
| 244 | continue |
| 245 | if mtime > cutoff: |
| 246 | continue |
| 247 | entry["mtime"] = mtime |
| 248 | dead_notes.append(entry) |
| 249 | |
| 250 | return { |
| 251 | "total_notes_indexed": index.notes_indexed, |
| 252 | "dead_count": len(dead_notes), |
| 253 | "dead_notes": dead_notes, |
| 254 | } |
| 255 | |
| 256 | |
| 257 | # --------------------------------------------------------------------------- |
| 258 | # clones — duplicate notes by content hash |
| 259 | # --------------------------------------------------------------------------- |
| 260 | |
| 261 | |
| 262 | def note_clones( |
| 263 | manifest: dict[str, str], |
| 264 | ) -> dict[str, Any]: |
| 265 | """Find groups of notes sharing the same content hash (exact duplicates). |
| 266 | |
| 267 | Args: |
| 268 | manifest: Snapshot manifest (path → content_hash). |
| 269 | |
| 270 | Returns: |
| 271 | Dict with keys: |
| 272 | |
| 273 | - ``total_notes``: int |
| 274 | - ``clone_groups``: list of ``{hash, paths}`` dicts, where each |
| 275 | group has 2+ paths sharing the same hash. |
| 276 | - ``total_clones``: int — total number of clone instances |
| 277 | (sum of len(paths) across all groups). |
| 278 | """ |
| 279 | notes = _notes_in_manifest(manifest) |
| 280 | by_hash: dict[str, list[str]] = defaultdict(list) |
| 281 | for path, content_hash in notes.items(): |
| 282 | by_hash[content_hash].append(path) |
| 283 | |
| 284 | groups = [ |
| 285 | {"hash": h, "paths": sorted(paths)} |
| 286 | for h, paths in by_hash.items() |
| 287 | if len(paths) > 1 |
| 288 | ] |
| 289 | groups.sort(key=lambda g: (-len(g["paths"]), g["paths"][0])) |
| 290 | |
| 291 | return { |
| 292 | "total_notes": len(notes), |
| 293 | "clone_groups": groups, |
| 294 | "total_clones": sum(len(g["paths"]) for g in groups), |
| 295 | } |
| 296 | |
| 297 | |
| 298 | # --------------------------------------------------------------------------- |
| 299 | # entangle — co-change note pairs |
| 300 | # --------------------------------------------------------------------------- |
| 301 | |
| 302 | |
| 303 | def note_entangle( |
| 304 | commits: list, |
| 305 | snapshot_loader: Any, |
| 306 | *, |
| 307 | min_co_changes: int = 2, |
| 308 | top: int = 20, |
| 309 | link_index: LinkIndex | None = None, |
| 310 | attachment_weight: int = 1, |
| 311 | ) -> dict[str, Any]: |
| 312 | """Find note pairs that change together in the same commits. |
| 313 | |
| 314 | Phase 3.5 extension: when *link_index* is provided, mist-attachment |
| 315 | co-references are added to the entanglement score. Two notes that |
| 316 | reference the same mist ID gain ``attachment_weight`` co-change credits. |
| 317 | |
| 318 | Args: |
| 319 | commits: List of CommitRecord, ordered newest-first. |
| 320 | snapshot_loader: Callable ``(commit_id) → manifest``. |
| 321 | min_co_changes: Minimum number of co-occurring commits for inclusion. |
| 322 | top: Maximum entries in ``ranking``. |
| 323 | link_index: Optional :class:`LinkIndex` for mist-attachment join. |
| 324 | attachment_weight: Credits awarded per shared mist attachment. |
| 325 | |
| 326 | Returns: |
| 327 | Dict with keys: |
| 328 | |
| 329 | - ``commits_analysed``: int |
| 330 | - ``attachment_pairs``: int — number of pairs sharing mist IDs (0 if |
| 331 | ``link_index`` is None). |
| 332 | - ``ranking``: list of ``{pair: [path_a, path_b], co_changes: int, |
| 333 | shared_attachments: int}`` dicts, sorted by co_changes desc then |
| 334 | pair lex asc. |
| 335 | """ |
| 336 | pair_counts: Counter[tuple[str, str]] = Counter() |
| 337 | attachment_pair_counts: Counter[tuple[str, str]] = Counter() |
| 338 | prev_manifest: dict[str, str] | None = None |
| 339 | |
| 340 | for commit in commits: |
| 341 | try: |
| 342 | current = snapshot_loader(commit.commit_id) or {} |
| 343 | except Exception: |
| 344 | continue |
| 345 | if prev_manifest is None: |
| 346 | prev_manifest = current |
| 347 | continue |
| 348 | changed_notes = sorted(_changed_notes_between(current, prev_manifest)) |
| 349 | for i, a in enumerate(changed_notes): |
| 350 | for b in changed_notes[i + 1:]: |
| 351 | pair_counts[(a, b)] += 1 |
| 352 | prev_manifest = current |
| 353 | |
| 354 | # Phase 3.5: add mist-attachment co-references to the same counter. |
| 355 | if link_index is not None: |
| 356 | for _mist_id, members in link_index.attachment_index.items(): |
| 357 | members_sorted = sorted(members) |
| 358 | for i, a in enumerate(members_sorted): |
| 359 | for b in members_sorted[i + 1:]: |
| 360 | attachment_pair_counts[(a, b)] += 1 |
| 361 | pair_counts[(a, b)] += attachment_weight |
| 362 | |
| 363 | pairs = [(pair, n) for pair, n in pair_counts.items() if n >= min_co_changes] |
| 364 | pairs.sort(key=lambda x: (-x[1], x[0])) |
| 365 | |
| 366 | return { |
| 367 | "commits_analysed": len(commits), |
| 368 | "attachment_pairs": len(attachment_pair_counts), |
| 369 | "ranking": [ |
| 370 | { |
| 371 | "pair": list(pair), |
| 372 | "co_changes": n, |
| 373 | "shared_attachments": attachment_pair_counts.get(pair, 0), |
| 374 | } |
| 375 | for pair, n in pairs[:top] |
| 376 | ], |
| 377 | } |
| 378 | |
| 379 | |
| 380 | # --------------------------------------------------------------------------- |
| 381 | # velocity — note additions per commit window |
| 382 | # --------------------------------------------------------------------------- |
| 383 | |
| 384 | |
| 385 | def note_velocity( |
| 386 | commits: list, |
| 387 | snapshot_loader: Any, |
| 388 | *, |
| 389 | window_size: int = 10, |
| 390 | ) -> dict[str, Any]: |
| 391 | """Compute note additions across two consecutive commit windows. |
| 392 | |
| 393 | Args: |
| 394 | commits: List of CommitRecord, ordered newest-first. |
| 395 | snapshot_loader: Callable ``(commit_id) → manifest``. |
| 396 | window_size: Number of commits per window. Two windows are compared. |
| 397 | |
| 398 | Returns: |
| 399 | Dict with keys: |
| 400 | |
| 401 | - ``window_size``: int |
| 402 | - ``recent_window``: ``{added, removed, modified, net}`` |
| 403 | - ``older_window``: ``{added, removed, modified, net}`` |
| 404 | - ``acceleration``: int — recent.net - older.net |
| 405 | """ |
| 406 | def _window_stats(window_commits: list) -> dict[str, int]: |
| 407 | """Compute add/remove/modify counts for a window.""" |
| 408 | if not window_commits: |
| 409 | return {"added": 0, "removed": 0, "modified": 0, "net": 0} |
| 410 | try: |
| 411 | newest = snapshot_loader(window_commits[0].commit_id) or {} |
| 412 | oldest = snapshot_loader(window_commits[-1].commit_id) or {} |
| 413 | except Exception: |
| 414 | return {"added": 0, "removed": 0, "modified": 0, "net": 0} |
| 415 | newest_notes = _notes_in_manifest(newest) |
| 416 | oldest_notes = _notes_in_manifest(oldest) |
| 417 | added = len(set(newest_notes) - set(oldest_notes)) |
| 418 | removed = len(set(oldest_notes) - set(newest_notes)) |
| 419 | modified = sum( |
| 420 | 1 for p in (set(newest_notes) & set(oldest_notes)) |
| 421 | if newest_notes[p] != oldest_notes[p] |
| 422 | ) |
| 423 | return { |
| 424 | "added": added, |
| 425 | "removed": removed, |
| 426 | "modified": modified, |
| 427 | "net": added - removed, |
| 428 | } |
| 429 | |
| 430 | if window_size <= 0: |
| 431 | raise ValueError(f"velocity: window_size must be positive, got {window_size}") |
| 432 | |
| 433 | recent = commits[:window_size] |
| 434 | older = commits[window_size:window_size * 2] |
| 435 | recent_stats = _window_stats(recent) |
| 436 | older_stats = _window_stats(older) |
| 437 | |
| 438 | return { |
| 439 | "window_size": window_size, |
| 440 | "recent_window": recent_stats, |
| 441 | "older_window": older_stats, |
| 442 | "acceleration": recent_stats["net"] - older_stats["net"], |
| 443 | } |
| 444 | |
| 445 | |
| 446 | # --------------------------------------------------------------------------- |
| 447 | # Convenience: walk commits + snapshot loader factory |
| 448 | # --------------------------------------------------------------------------- |
| 449 | |
| 450 | |
| 451 | def make_snapshot_loader(root: pathlib.Path): |
| 452 | """Return a closure ``(commit_id) → manifest`` for use with these helpers. |
| 453 | |
| 454 | Args: |
| 455 | root: Repository root. |
| 456 | |
| 457 | Returns: |
| 458 | Callable that returns the snapshot manifest for *commit_id*, or an |
| 459 | empty dict on failure. |
| 460 | """ |
| 461 | from muse.core.store import get_commit_snapshot_manifest |
| 462 | |
| 463 | def _loader(commit_id: str) -> dict[str, str]: |
| 464 | try: |
| 465 | return get_commit_snapshot_manifest(root, commit_id) or {} |
| 466 | except Exception: |
| 467 | return {} |
| 468 | |
| 469 | return _loader |
File History
2 commits
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6
chore(tests): add docstring to tests/__init__.py so rc14 tr…
Human
26 days ago
sha256:d8316ffae901be06347e16ab55be11868eb519dd16ade3e8aa16a99e662f7e62
baseline: rc14 re-baseline after rc3 store corruption recovery
Human
patch
26 days ago