aaronrene / gabriel-muse public
velocity.py python
612 lines 22.1 KB
Raw
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6 chore(tests): add docstring to tests/__init__.py so rc14 tr… Human 33 days ago
1 """muse code velocity — symbol-growth rate by module, with acceleration and next-change prediction.
2
3 This is the *energy map* of a codebase.
4
5 ``muse code hotspots`` tells you which symbols changed most.
6 ``muse code velocity`` tells you where the codebase is growing, where it is
7 shrinking, where it has stalled — and how fast each trend is accelerating.
8
9 It also answers a forward-looking question: **which symbols are most likely to
10 change in the next few commits?** This is computed statistically from
11 recency, frequency, and module acceleration.
12
13 Why this matters
14 ----------------
15 File commit counts or line-change counts obscure the signal. A file with 500
16 line changes might just be a re-format. A module that gained 12 new symbols
17 across 8 commits is actively expanding its API surface — that is genuine
18 architectural investment.
19
20 Three windows
21 -------------
22 Velocity is computed over two consecutive commit windows of equal size
23 (``--window N``, default 20):
24
25 * **Current window** — the most recent N commits.
26 * **Prior window** — the N commits before that.
27
28 The difference between the two gives **acceleration**:
29
30 * Positive acceleration = module growing faster than before.
31 * Negative acceleration = module is decelerating / winding down.
32 * Zero acceleration from zero velocity = stagnation.
33
34 Prediction
35 ----------
36 ``--predict K`` outputs the top K symbols most likely to change in the next
37 commit, ranked by a composite score:
38
39 ``score = frequency × recency_weight × module_velocity_weight``
40
41 where ``recency_weight = 1 / (1 + rank)`` (rank 0 = most recent commit).
42 This is a statistical signal, not a guarantee.
43
44 Usage::
45
46 muse code velocity
47 muse code velocity --window 10 --top 10
48 muse code velocity --predict 5
49 muse code velocity --since v1.0
50 muse code velocity --json
51
52 Output::
53
54 Symbol velocity — HEAD (40 commits · window: 20)
55 Sorted by: net growth, current window
56
57 MODULE ADD DEL NET MOD ACCEL BAR
58 muse/core/ +14 -2 +12 47 ▲ +4 ████████████
59 muse/cli/commands/ +11 -1 +10 31 ▲ +2 ██████████
60 tests/ +8 -0 +8 12 ▲ +3 ████████
61 muse/plugins/ +3 -5 -2 8 ▼ -1 ▏ (net negative)
62 docs/ 0 -0 0 0 ─ (stagnant 15 commits)
63
64 Acceleration leaders: muse/core/ (+4 net vs prior window)
65 Stagnant modules: docs/ (0 changes in 18 commits)
66
67 --predict output::
68
69 Next-change predictions (top 5 by statistical likelihood):
70 1 muse/core/store.py::resolve_commit_ref score: 0.91
71 2 muse/cli/commands/dead.py::run score: 0.84
72 3 tests/test_code_commands.py::TestHotspots score: 0.71
73
74 JSON output (--json)::
75
76 {
77 "ref": "HEAD",
78 "window_size": 20,
79 "commits_analysed": 40,
80 "truncated": false,
81 "filters": { "top": 20, "since": null },
82 "modules": [
83 {
84 "module": "muse/core/",
85 "current": { "added": 14, "removed": 2, "net": 12, "modified": 47, "active_commits": 18 },
86 "prior": { "added": 10, "removed": 2, "net": 8, "modified": 33, "active_commits": 14 },
87 "acceleration": 4,
88 "stagnant_commits": 0
89 }
90 ],
91 "predictions": [
92 { "address": "muse/core/store.py::resolve_commit_ref", "score": 0.91 }
93 ]
94 }
95 """
96
97 from __future__ import annotations
98
99 import argparse
100 import json
101 import logging
102 import pathlib
103 import sys
104 from dataclasses import dataclass, field
105 from typing import TypedDict
106
107 from muse.core.errors import ExitCode
108 from muse.core.repo import read_repo_id, require_repo
109 from muse.core.store import read_current_branch, resolve_commit_ref
110 from muse.domain import DomainOp
111 from muse.plugins.code._query import flat_symbol_ops, walk_commits_bfs
112 from muse.core.validation import clamp_int, sanitize_display
113
114 type _CounterMap = dict[str, int]
115 type _ModAccumMap = dict[str, "_ModuleAccumulator"]
116 type _SymFreqMap = dict[str, "_SymbolFreq"]
117
118 logger = logging.getLogger(__name__)
119
120 # ── Constants ──────────────────────────────────────────────────────────────────
121
122 _DEFAULT_WINDOW = 20
123 _DEFAULT_TOP = 20
124 _DEFAULT_MAX_COMMITS = 10_000
125 _DEFAULT_PREDICT = 0 # 0 = disabled
126 _BAR_WIDTH = 20 # max bar width in characters
127
128 # Commits with more than this many symbol ops are mass-refactors — skip for
129 # module velocity (they would unfairly spike a module's counts).
130 _MAX_OPS_PER_COMMIT = 500
131
132
133 # ── Helpers ────────────────────────────────────────────────────────────────────
134
135
136
137 def _module_of(file_path: str) -> str:
138 """Return the containing directory of a file path.
139
140 ``muse/core/store.py`` → ``muse/core/``
141 ``tests/test_billing.py`` → ``tests/``
142 ``billing.py`` → ``(root)``
143 """
144 parts = file_path.replace("\\", "/").rsplit("/", 1)
145 if len(parts) == 1:
146 return "(root)"
147 return parts[0] + "/"
148
149
150 def _bar(net: int, max_abs: int) -> str:
151 """Return a unicode block bar proportional to *net* / *max_abs*."""
152 if max_abs == 0:
153 return ""
154 ratio = abs(net) / max_abs
155 filled = round(ratio * _BAR_WIDTH)
156 bar = "█" * filled
157 if net < 0:
158 bar = bar or "▏"
159 return f"{bar} (net negative)"
160 return bar or "▏"
161
162
163 # ── Data types ─────────────────────────────────────────────────────────────────
164
165
166 @dataclass
167 class _WindowStats:
168 added: int = 0
169 removed: int = 0
170 modified: int = 0
171 active_commits: int = 0
172
173 @property
174 def net(self) -> int:
175 return self.added - self.removed
176
177
178 @dataclass
179 class _ModuleAccumulator:
180 current: _WindowStats = field(default_factory=_WindowStats)
181 prior: _WindowStats = field(default_factory=_WindowStats)
182 last_active_rank: int = -1 # commit rank (0 = HEAD) of last activity
183 stagnant_commits: int = 0 # consecutive commits with no activity
184
185
186 class _ModuleOut(TypedDict):
187 module: str
188 current: _CounterMap
189 prior: _CounterMap
190 acceleration: int
191 stagnant_commits: int
192
193
194 class _PredictionOut(TypedDict):
195 address: str
196 module: str
197 score: float
198 frequency: int
199 last_commit_rank: int
200
201
202 # ── Core algorithm ─────────────────────────────────────────────────────────────
203
204
205 @dataclass
206 class _SymbolFreq:
207 frequency: int = 0
208 last_rank: int = 0 # 0 = most recent commit
209 module: str = ""
210
211
212 def _walk_and_collect(
213 root: pathlib.Path,
214 head_commit_id: str,
215 stop_at: str | None,
216 window_size: int,
217 max_commits: int,
218 ) -> tuple[
219 dict[str, _ModuleAccumulator], # per-module stats
220 dict[str, _SymbolFreq], # per-symbol frequency (current window only)
221 int, # total commits analysed
222 bool, # truncated
223 ]:
224 """Single BFS pass building module velocity and symbol frequency data."""
225 commits, truncated = walk_commits_bfs(
226 root, head_commit_id, max_commits, stop_at_commit_id=stop_at
227 )
228
229 modules: _ModAccumMap = {}
230 symbol_freq: _SymFreqMap = {}
231
232 # Track which modules were active per commit (for stagnation detection).
233 # commits are sorted newest-first after BFS.
234 for rank, commit in enumerate(commits):
235 if commit.structured_delta is None:
236 continue
237 ops: list[DomainOp] = commit.structured_delta["ops"]
238
239 # Gather all leaf symbol ops for this commit.
240 all_ops = list(flat_symbol_ops(ops))
241
242 # Skip mass-refactor commits.
243 if len(all_ops) > _MAX_OPS_PER_COMMIT:
244 continue
245
246 # Modules that had activity in this commit (for stagnation tracking).
247 active_modules: set[str] = set()
248
249 for op in all_ops:
250 addr: str = op["address"]
251 if "::import::" in addr:
252 continue
253
254 file_path = addr.split("::")[0]
255 mod = _module_of(file_path)
256 active_modules.add(mod)
257 acc = modules.setdefault(mod, _ModuleAccumulator())
258
259 op_kind = op.get("op", "")
260
261 # Determine which window this commit belongs to.
262 in_current = rank < window_size
263 in_prior = window_size <= rank < 2 * window_size
264
265 if in_current:
266 if op_kind == "insert":
267 acc.current.added += 1
268 elif op_kind == "delete":
269 acc.current.removed += 1
270 elif op_kind == "replace":
271 acc.current.modified += 1
272 elif in_prior:
273 if op_kind == "insert":
274 acc.prior.added += 1
275 elif op_kind == "delete":
276 acc.prior.removed += 1
277 elif op_kind == "replace":
278 acc.prior.modified += 1
279
280 # Symbol frequency (current window only) for prediction.
281 if in_current:
282 sf = symbol_freq.setdefault(addr, _SymbolFreq(module=mod))
283 sf.frequency += 1
284 if sf.last_rank == 0 or rank < sf.last_rank:
285 sf.last_rank = rank
286
287 # Track active_commits per window.
288 for mod in active_modules:
289 acc = modules.setdefault(mod, _ModuleAccumulator())
290 if rank < window_size:
291 acc.current.active_commits += 1
292 elif rank < 2 * window_size:
293 acc.prior.active_commits += 1
294 # Update last active rank.
295 if acc.last_active_rank < 0 or rank < acc.last_active_rank:
296 acc.last_active_rank = rank
297
298 # Compute stagnant_commits: how many leading commits (from HEAD) had
299 # zero activity for this module.
300 for mod, acc in modules.items():
301 if acc.last_active_rank < 0:
302 acc.stagnant_commits = len(commits)
303 else:
304 acc.stagnant_commits = acc.last_active_rank
305
306 return modules, symbol_freq, len(commits), truncated
307
308
309 # ── Prediction ─────────────────────────────────────────────────────────────────
310
311
312 def _compute_predictions(
313 symbol_freq: _SymFreqMap,
314 modules: _ModAccumMap,
315 window_size: int,
316 top_k: int,
317 ) -> list[_PredictionOut]:
318 """Score each symbol in the current window and return top-K predictions.
319
320 Score = frequency × recency_weight × module_velocity_weight
321
322 recency_weight = 1 / (1 + last_commit_rank)
323 (0 = most recent commit → weight 1.0)
324 module_velocity = max(0, net_current) (growth modules get a boost)
325 module_vel_weight = 1.0 + normalised module velocity
326 """
327 if not symbol_freq or top_k <= 0:
328 return []
329
330 # Normalise module velocity (0..1 range).
331 max_net = max(
332 (max(0, modules[sf.module].current.net) for sf in symbol_freq.values() if sf.module in modules),
333 default=0,
334 ) or 1
335
336 scored: list[tuple[float, _PredictionOut]] = []
337 for addr, sf in symbol_freq.items():
338 if sf.frequency == 0:
339 continue
340 recency_w = 1.0 / (1.0 + sf.last_rank)
341 mod_net = max(0, modules[sf.module].current.net) if sf.module in modules else 0
342 mod_w = 1.0 + (mod_net / max_net) # 1.0 .. 2.0
343 score = round(sf.frequency * recency_w * mod_w, 4)
344
345 scored.append((score, _PredictionOut(
346 address=addr,
347 module=sf.module,
348 score=score,
349 frequency=sf.frequency,
350 last_commit_rank=sf.last_rank,
351 )))
352
353 scored.sort(key=lambda t: -t[0])
354 return [out for _, out in scored[:top_k]]
355
356
357 # ── Formatters ─────────────────────────────────────────────────────────────────
358
359
360 def _print_table(
361 ranked: list[tuple[str, _ModuleAccumulator]],
362 predictions: list[_PredictionOut],
363 ref: str,
364 commits_analysed: int,
365 window_size: int,
366 truncated: bool,
367 since: str | None,
368 ) -> None:
369 scope = f"{since}..{ref}" if since else ref
370 trunc = " ⚠️ truncated" if truncated else ""
371 print(
372 f"\nSymbol velocity — {scope}"
373 f" ({commits_analysed} commits · window: {window_size}{trunc})"
374 )
375 print("Sorted by: net growth, current window\n")
376
377 if not ranked:
378 print(" (no modules with symbol-level changes found)")
379 return
380
381 max_abs = max(abs(acc.current.net) for _, acc in ranked) or 1
382 max_mod = max(len(mod) for mod, _ in ranked)
383
384 hdr = (
385 f" {'MODULE':<{max_mod}} {'ADD':>5} {'DEL':>5} {'NET':>5} "
386 f"{'MOD':>5} {'ACCEL':>7} BAR"
387 )
388 print(hdr)
389 print(" " + "─" * (len(hdr) - 2))
390
391 accel_leaders: list[str] = []
392 stagnant: list[tuple[str, int]] = []
393
394 for mod, acc in ranked:
395 accel = acc.current.net - acc.prior.net
396 if accel > 0:
397 accel_str = f"▲ +{accel}"
398 elif accel < 0:
399 accel_str = f"▼ {accel}"
400 else:
401 accel_str = "─"
402
403 bar = _bar(acc.current.net, max_abs)
404
405 add_str = f"+{acc.current.added}" if acc.current.added else "0"
406 del_str = f"-{acc.current.removed}" if acc.current.removed else "0"
407 net_str = f"+{acc.current.net}" if acc.current.net > 0 else str(acc.current.net)
408
409 stag = acc.stagnant_commits
410 if stag > 0:
411 stagnant.append((mod, stag))
412 note = f" (stagnant {stag} commit{'s' if stag != 1 else ''})"
413 print(
414 f" {mod:<{max_mod}} {add_str:>5} {del_str:>5} "
415 f"{net_str:>5} {acc.current.modified:>5} {accel_str:>7} {note}"
416 )
417 else:
418 print(
419 f" {mod:<{max_mod}} {add_str:>5} {del_str:>5} "
420 f"{net_str:>5} {acc.current.modified:>5} {accel_str:>7} {bar}"
421 )
422
423 if accel >= 2:
424 accel_leaders.append(f"{mod} (+{accel} net vs prior window)")
425
426 print("")
427 if accel_leaders:
428 print(f"Acceleration leaders: {', '.join(accel_leaders[:3])}")
429 stag_str = ", ".join(f"{m} ({n} commits)" for m, n in stagnant[:3])
430 if stag_str:
431 print(f"Stagnant modules: {stag_str}")
432
433 if predictions:
434 print(f"\nNext-change predictions (top {len(predictions)}):")
435 for i, pred in enumerate(predictions, 1):
436 print(f" {i:>2} {sanitize_display(pred['address']):<60} score: {pred['score']:.2f}")
437
438
439 # ── CLI ────────────────────────────────────────────────────────────────────────
440
441
442 def register(
443 subparsers: "argparse._SubParsersAction[argparse.ArgumentParser]",
444 ) -> None:
445 """Register the velocity subcommand."""
446 parser = subparsers.add_parser(
447 "velocity",
448 help=(
449 "Symbol-growth rate by module — where the codebase is growing, "
450 "shrinking, accelerating, or stagnating."
451 ),
452 description=__doc__,
453 formatter_class=argparse.RawDescriptionHelpFormatter,
454 )
455 parser.add_argument(
456 "--window", "-w",
457 type=int, default=_DEFAULT_WINDOW, metavar="N",
458 help=(
459 f"Commits per analysis window (default: {_DEFAULT_WINDOW}). "
460 f"Two consecutive windows are compared to compute acceleration."
461 ),
462 )
463 parser.add_argument(
464 "--top", "-n",
465 type=int, default=_DEFAULT_TOP, metavar="N",
466 help=f"Number of modules to show (default: {_DEFAULT_TOP}).",
467 )
468 parser.add_argument(
469 "--predict", "-p",
470 type=int, default=_DEFAULT_PREDICT, metavar="K",
471 dest="predict",
472 help=(
473 "Show the top K symbols most likely to change in the next commit "
474 "(default: 0 = disabled). "
475 "Ranked by: recency × frequency × module-velocity."
476 ),
477 )
478 parser.add_argument(
479 "--since", "-s",
480 default=None, metavar="REF",
481 help="Limit analysis to commits reachable from HEAD but not from REF.",
482 )
483 parser.add_argument(
484 "--max-commits",
485 type=int, default=_DEFAULT_MAX_COMMITS, metavar="N",
486 dest="max_commits",
487 help=f"Maximum commits to scan (default: {_DEFAULT_MAX_COMMITS}).",
488 )
489 parser.add_argument(
490 "--json",
491 action="store_true", dest="as_json",
492 help="Emit results as JSON.",
493 )
494 parser.set_defaults(func=run)
495
496
497 def run(args: argparse.Namespace) -> None:
498 """Compute symbol-growth velocity by module.
499
500 Mines the commit history for symbol insert/delete/modify operations and
501 aggregates them by module (containing directory). Compares the current
502 window to the prior window to detect acceleration and stagnation.
503
504 The ``--predict K`` flag uses recency, frequency, and module velocity to
505 rank the top K symbols most likely to change in the next commit.
506 """
507 window: int = clamp_int(args.window, 1, 1000, 'window')
508 top: int = clamp_int(args.top, 1, 10_000, 'top')
509 predict_k: int = clamp_int(args.predict, 0, 1000, 'predict_k')
510 since: str | None = args.since
511 max_commits: int = clamp_int(args.max_commits, 1, 100_000, 'max_commits')
512 as_json: bool = args.as_json
513
514 # ── Validation ────────────────────────────────────────────────────────────
515 if window < 1:
516 print("❌ --window must be >= 1.", file=sys.stderr)
517 raise SystemExit(ExitCode.USER_ERROR)
518 if top < 1:
519 print("❌ --top must be >= 1.", file=sys.stderr)
520 raise SystemExit(ExitCode.USER_ERROR)
521 if predict_k < 0:
522 print("❌ --predict must be >= 0.", file=sys.stderr)
523 raise SystemExit(ExitCode.USER_ERROR)
524 if max_commits < 1:
525 print("❌ --max-commits must be >= 1.", file=sys.stderr)
526 raise SystemExit(ExitCode.USER_ERROR)
527 # Need at least 2 windows to compute acceleration.
528 effective_max = max(max_commits, window * 2)
529
530 # ── Repo setup ────────────────────────────────────────────────────────────
531 root = require_repo()
532
533 from muse.plugins.registry import read_domain
534 try:
535 if read_domain(root) == "knowtation":
536 from muse.plugins.knowtation.cli_hooks import run_velocity_for_vault
537 run_velocity_for_vault(root, args)
538 return
539 except Exception: # noqa: BLE001
540 pass
541
542 repo_id = read_repo_id(root)
543 branch = read_current_branch(root)
544
545 head = resolve_commit_ref(root, repo_id, branch, None)
546 if head is None:
547 print("❌ HEAD commit not found.", file=sys.stderr)
548 raise SystemExit(ExitCode.USER_ERROR)
549
550 stop_at: str | None = None
551 if since is not None:
552 since_commit = resolve_commit_ref(root, repo_id, branch, since)
553 if since_commit is None:
554 print(f"❌ Commit '{since}' not found.", file=sys.stderr)
555 raise SystemExit(ExitCode.USER_ERROR)
556 stop_at = since_commit.commit_id
557
558 # ── Main pass ─────────────────────────────────────────────────────────────
559 modules, symbol_freq, commits_analysed, truncated = _walk_and_collect(
560 root, head.commit_id, stop_at, window, effective_max
561 )
562
563 # ── Rank modules by current-window net growth ──────────────────────────────
564 ranked = sorted(
565 modules.items(),
566 key=lambda kv: (-kv[1].current.net, -kv[1].current.modified, kv[0]),
567 )[:top]
568
569 # ── Predictions ───────────────────────────────────────────────────────────
570 predictions = _compute_predictions(symbol_freq, modules, window, predict_k)
571
572 # ── Output ────────────────────────────────────────────────────────────────
573 if as_json:
574 modules_out: list[_ModuleOut] = [
575 _ModuleOut(
576 module=mod,
577 current={
578 "added": acc.current.added,
579 "removed": acc.current.removed,
580 "net": acc.current.net,
581 "modified": acc.current.modified,
582 "active_commits": acc.current.active_commits,
583 },
584 prior={
585 "added": acc.prior.added,
586 "removed": acc.prior.removed,
587 "net": acc.prior.net,
588 "modified": acc.prior.modified,
589 "active_commits": acc.prior.active_commits,
590 },
591 acceleration=acc.current.net - acc.prior.net,
592 stagnant_commits=acc.stagnant_commits,
593 )
594 for mod, acc in ranked
595 ]
596 print(json.dumps({
597 "ref": branch,
598 "window_size": window,
599 "commits_analysed": commits_analysed,
600 "truncated": truncated,
601 "filters": {
602 "top": top,
603 "since": since,
604 "predict": predict_k,
605 "max_commits": max_commits,
606 },
607 "modules": [dict(m) for m in modules_out],
608 "predictions": [dict(p) for p in predictions],
609 }, indent=2))
610 return
611
612 _print_table(ranked, predictions, branch, commits_analysed, window, truncated, since)
File History 2 commits
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6 chore(tests): add docstring to tests/__init__.py so rc14 tr… Human 33 days ago
sha256:d8316ffae901be06347e16ab55be11868eb519dd16ade3e8aa16a99e662f7e62 baseline: rc14 re-baseline after rc3 store corruption recovery Human patch 33 days ago