repo_card_enrichment.py
python
sha256:73f24973678ceefd56628ee17c6d0535d86e33533828af6c63348f50941515ad
Merge branch 'fix/smoke-test-tag-label' into dev
Human
16 days ago
| 1 | """ |
| 2 | repo_card_enrichment.py |
| 3 | ======================= |
| 4 | |
| 5 | Single-pass enrichment service for the enriched repo card component. |
| 6 | |
| 7 | All five signals are derived from existing postgres tables — no engine |
| 8 | subprocess calls, no per-card HTTP round trips. The public entry point |
| 9 | ``enrich_repo_cards()`` issues five batched queries (one per signal) and |
| 10 | merges the results into a ``RepoCardEnrichment`` per repo_id. |
| 11 | |
| 12 | Signal inventory |
| 13 | ---------------- |
| 14 | pulse_buckets list[PulseBucket] 30 daily commit-count buckets, normalised to SVG height |
| 15 | autonomy_pct int 0-100, % commits with a non-empty agent_id |
| 16 | hottest_symbol SymbolStat | None symbol with highest churn_30d in musehub_symbol_intel |
| 17 | blast_leader SymbolStat | None symbol with highest blast score in musehub_symbol_intel |
| 18 | dead_count int high-confidence dead symbols in musehub_intel_dead |
| 19 | error_count int breakage errors from musehub_intel_breakage_meta |
| 20 | warning_count int breakage warnings from musehub_intel_breakage_meta |
| 21 | health_status 'clean' | 'warn' | 'risk' derived property — no extra query |
| 22 | |
| 23 | Spectral colour scheme |
| 24 | ---------------------- |
| 25 | Pulse bar colours are interpolated along the spectral gradient defined in the |
| 26 | CSS design token ``--gradient-spectral``: |
| 27 | |
| 28 | blue #60a5fa → indigo #818cf8 → violet #a78bfa → pink #c084fc |
| 29 | |
| 30 | Bar i of N is assigned the hex colour at position i/(N-1) along this gradient. |
| 31 | The interpolation is pure Python so the colour ends up as a static ``fill`` |
| 32 | attribute in the inline SVG — no CSS gradient needed on the bars. |
| 33 | |
| 34 | Usage |
| 35 | ----- |
| 36 | enrichments = await enrich_repo_cards(db, repo_ids) |
| 37 | card_data = enrichments.get(repo_id) # RepoCardEnrichment | None |
| 38 | """ |
| 39 | from __future__ import annotations |
| 40 | |
| 41 | import colorsys |
| 42 | from dataclasses import dataclass, field |
| 43 | from datetime import date, datetime, timedelta, timezone |
| 44 | |
| 45 | import sqlalchemy as sa |
| 46 | from sqlalchemy import case, func, select |
| 47 | from sqlalchemy.ext.asyncio import AsyncSession |
| 48 | |
| 49 | from musehub.db.musehub_intel_models import ( |
| 50 | MusehubIntelBreakageMeta, |
| 51 | MusehubIntelDead, |
| 52 | MusehubSymbolIntel, |
| 53 | ) |
| 54 | from musehub.db.musehub_repo_models import MusehubCommit, MusehubCommitRef |
| 55 | from musehub.types.json_types import IntDict |
| 56 | |
| 57 | # --------------------------------------------------------------------------- |
| 58 | # Spectral palette — four stops from the --gradient-spectral CSS token |
| 59 | # --------------------------------------------------------------------------- |
| 60 | |
| 61 | _SPECTRAL_STOPS: list[tuple[int, int, int]] = [ |
| 62 | (0x60, 0xA5, 0xFA), # #60a5fa blue |
| 63 | (0x81, 0x8C, 0xF8), # #818cf8 indigo |
| 64 | (0xA7, 0x8B, 0xFA), # #a78bfa violet |
| 65 | (0xC0, 0x84, 0xFC), # #c084fc pink |
| 66 | ] |
| 67 | |
| 68 | |
| 69 | def _lerp(a: float, b: float, t: float) -> float: |
| 70 | """Linear interpolation between a and b at position t ∈ [0.0, 1.0].""" |
| 71 | return a + (b - a) * t |
| 72 | |
| 73 | |
| 74 | def spectral_color(index: int, total: int) -> str: |
| 75 | """ |
| 76 | Return the hex colour for bar ``index`` of ``total`` bars along the |
| 77 | spectral gradient. |
| 78 | |
| 79 | The gradient is divided into ``len(_SPECTRAL_STOPS) - 1`` equal segments. |
| 80 | ``index`` is first mapped to a position t ∈ [0.0, 1.0] across the full |
| 81 | gradient, then the surrounding two stops are interpolated. |
| 82 | |
| 83 | Parameters |
| 84 | ---------- |
| 85 | index: 0-based bar index |
| 86 | total: total number of bars (must be ≥ 1) |
| 87 | |
| 88 | Returns |
| 89 | ------- |
| 90 | Hex colour string, e.g. ``"#60a5fa"``. |
| 91 | """ |
| 92 | if total <= 1: |
| 93 | r, g, b = _SPECTRAL_STOPS[0] |
| 94 | return f"#{r:02x}{g:02x}{b:02x}" |
| 95 | |
| 96 | t = index / (total - 1) # position 0.0 → 1.0 |
| 97 | n_segments = len(_SPECTRAL_STOPS) - 1 |
| 98 | segment = min(int(t * n_segments), n_segments - 1) # which segment |
| 99 | local_t = (t * n_segments) - segment # position within segment |
| 100 | |
| 101 | lo = _SPECTRAL_STOPS[segment] |
| 102 | hi = _SPECTRAL_STOPS[segment + 1] |
| 103 | |
| 104 | r = round(_lerp(lo[0], hi[0], local_t)) |
| 105 | g = round(_lerp(lo[1], hi[1], local_t)) |
| 106 | b = round(_lerp(lo[2], hi[2], local_t)) |
| 107 | return f"#{r:02x}{g:02x}{b:02x}" |
| 108 | |
| 109 | |
| 110 | # --------------------------------------------------------------------------- |
| 111 | # Data models |
| 112 | # --------------------------------------------------------------------------- |
| 113 | |
| 114 | _SPARKLINE_HEIGHT = 24 # max SVG bar height in px units |
| 115 | _PULSE_DAYS = 30 # window for pulse sparkline |
| 116 | |
| 117 | |
| 118 | @dataclass |
| 119 | class PulseBucket: |
| 120 | """ |
| 121 | One day's commit count, pre-normalised to an SVG bar height. |
| 122 | |
| 123 | Attributes |
| 124 | ---------- |
| 125 | date: ISO date string (YYYY-MM-DD) |
| 126 | count: raw commit count for this day |
| 127 | h: bar height in SVG user units, scaled so the busiest day = _SPARKLINE_HEIGHT |
| 128 | color: hex colour — always #60a5fa (domain-code blue), matching the profile heatmap |
| 129 | opacity: float in [0, 1] — encodes commit intensity; 0.1 for empty days, |
| 130 | scaling from 0.25 → 1.0 proportionally for days with commits |
| 131 | """ |
| 132 | date: str |
| 133 | count: int |
| 134 | h: int |
| 135 | color: str |
| 136 | opacity: float = 1.0 |
| 137 | |
| 138 | |
| 139 | @dataclass |
| 140 | class SymbolStat: |
| 141 | """ |
| 142 | Compact symbol reference for display in a repo card intel row. |
| 143 | |
| 144 | Attributes |
| 145 | ---------- |
| 146 | address: full Muse symbol address, e.g. "musehub/services/foo.py::bar" |
| 147 | name: short display name — the last segment after ``::`` |
| 148 | churn_30d: times this symbol was changed in the last 30 days |
| 149 | blast: total downstream symbol count (blast radius) |
| 150 | """ |
| 151 | address: str |
| 152 | churn_30d: int |
| 153 | blast: int |
| 154 | |
| 155 | @property |
| 156 | def name(self) -> str: |
| 157 | """Return the symbol name — the part after the last ``::``.""" |
| 158 | return self.address.rsplit("::", 1)[-1] if "::" in self.address else self.address |
| 159 | |
| 160 | |
| 161 | @dataclass |
| 162 | class RepoCardEnrichment: |
| 163 | """ |
| 164 | Complete enrichment payload for one repo card. |
| 165 | |
| 166 | Constructed by ``enrich_repo_cards()`` in a single batched pass. |
| 167 | All fields have safe defaults so a card always renders even when intel |
| 168 | tables contain no rows for a given repo. |
| 169 | |
| 170 | The ``health_status`` property is a derived signal — it requires no |
| 171 | additional query. |
| 172 | |
| 173 | Attributes |
| 174 | ---------- |
| 175 | repo_id: sha256-prefixed repo identifier |
| 176 | pulse_buckets: 30 PulseBucket entries covering the last 30 days, |
| 177 | always length 30 (days with zero commits are zero-filled) |
| 178 | autonomy_pct: percentage of commits carrying a non-empty agent_id; |
| 179 | 0 when the repo has no commits at all |
| 180 | hottest_symbol: the symbol with the highest churn_30d, or None if |
| 181 | musehub_symbol_intel has no rows for this repo |
| 182 | blast_leader: the symbol with the highest blast score, or None if |
| 183 | musehub_symbol_intel has no rows for this repo |
| 184 | dead_count: count of high-confidence dead symbols |
| 185 | error_count: count of breakage errors from musehub_intel_breakage_meta |
| 186 | warning_count: count of breakage warnings from musehub_intel_breakage_meta |
| 187 | """ |
| 188 | repo_id: str |
| 189 | pulse_buckets: list[PulseBucket] = field(default_factory=list) |
| 190 | autonomy_pct: int = 0 |
| 191 | hottest_symbol: SymbolStat | None = None |
| 192 | blast_leader: SymbolStat | None = None |
| 193 | dead_count: int = 0 |
| 194 | error_count: int = 0 |
| 195 | warning_count: int = 0 |
| 196 | |
| 197 | @property |
| 198 | def health_status(self) -> str: |
| 199 | """ |
| 200 | Three-tier health classification derived from dead + breakage signals. |
| 201 | |
| 202 | Returns |
| 203 | ------- |
| 204 | 'risk' — any breakage errors are present (highest severity) |
| 205 | 'warn' — dead symbols or breakage warnings present, no errors |
| 206 | 'clean' — zero dead symbols and zero breakage issues |
| 207 | """ |
| 208 | if self.error_count > 0: |
| 209 | return "risk" |
| 210 | if self.dead_count > 0 or self.warning_count > 0: |
| 211 | return "warn" |
| 212 | return "clean" |
| 213 | |
| 214 | |
| 215 | RepoCardMap = dict[str, RepoCardEnrichment] |
| 216 | """Maps repo_id → enrichment; returned by ``enrich_repo_cards``.""" |
| 217 | |
| 218 | |
| 219 | # --------------------------------------------------------------------------- |
| 220 | # Internal helpers |
| 221 | # --------------------------------------------------------------------------- |
| 222 | |
| 223 | _PULSE_COLOR = "#60a5fa" # domain-code blue — matches the profile activity heatmap |
| 224 | |
| 225 | |
| 226 | def _make_empty_buckets(today: date) -> list[PulseBucket]: |
| 227 | """Return 30 zero-count PulseBuckets covering today and the 29 days before.""" |
| 228 | buckets = [] |
| 229 | for i in range(_PULSE_DAYS): |
| 230 | day = today - timedelta(days=_PULSE_DAYS - 1 - i) |
| 231 | buckets.append(PulseBucket( |
| 232 | date=day.isoformat(), |
| 233 | count=0, |
| 234 | h=0, |
| 235 | color=_PULSE_COLOR, |
| 236 | opacity=0.1, |
| 237 | )) |
| 238 | return buckets |
| 239 | |
| 240 | |
| 241 | def _normalise_heights(buckets: list[PulseBucket]) -> None: |
| 242 | """ |
| 243 | Scale bar heights and opacity in-place so the busiest bar = _SPARKLINE_HEIGHT px. |
| 244 | |
| 245 | Opacity mirrors the profile activity heatmap — it encodes commit intensity |
| 246 | using the same #60a5fa base at varying alpha: |
| 247 | |
| 248 | count == 0 → opacity 0.10 (dim placeholder, matches bg-inset feel) |
| 249 | count > 0 → opacity 0.25 – 1.0 (linear from quietest to busiest day) |
| 250 | |
| 251 | A minimum height of 2 px is applied to any bucket with count > 0 so that |
| 252 | even a single commit is visible. |
| 253 | """ |
| 254 | max_count = max((b.count for b in buckets), default=0) |
| 255 | if max_count == 0: |
| 256 | return |
| 257 | for bucket in buckets: |
| 258 | if bucket.count == 0: |
| 259 | bucket.h = 0 |
| 260 | bucket.opacity = 0.1 |
| 261 | else: |
| 262 | scaled = round((bucket.count / max_count) * _SPARKLINE_HEIGHT) |
| 263 | bucket.h = max(scaled, 2) |
| 264 | # Scale opacity 0.25 → 1.0 proportionally to count |
| 265 | bucket.opacity = round(0.25 + 0.75 * (bucket.count / max_count), 3) |
| 266 | |
| 267 | |
| 268 | def _build_pulse( |
| 269 | raw: dict[str, int], # date-string → commit count |
| 270 | today: date, |
| 271 | ) -> list[PulseBucket]: |
| 272 | """ |
| 273 | Merge raw day-count data into a complete 30-bucket sparkline. |
| 274 | |
| 275 | Parameters |
| 276 | ---------- |
| 277 | raw: mapping of ISO date string → commit count |
| 278 | today: the reference date for the 30-day window |
| 279 | """ |
| 280 | buckets = _make_empty_buckets(today) |
| 281 | for bucket in buckets: |
| 282 | bucket.count = raw.get(bucket.date, 0) |
| 283 | _normalise_heights(buckets) |
| 284 | return buckets |
| 285 | |
| 286 | |
| 287 | def _short_name(address: str) -> str: |
| 288 | """Extract the symbol name from a full address (part after last ``::``)..""" |
| 289 | return address.rsplit("::", 1)[-1] if "::" in address else address |
| 290 | |
| 291 | |
| 292 | # --------------------------------------------------------------------------- |
| 293 | # Public API |
| 294 | # --------------------------------------------------------------------------- |
| 295 | |
| 296 | async def enrich_repo_cards( |
| 297 | db: AsyncSession, |
| 298 | repo_ids: list[str], |
| 299 | ) -> RepoCardMap: |
| 300 | """ |
| 301 | Batch-enrich a list of repo_ids for display in the enriched repo card component. |
| 302 | |
| 303 | Issues five SQL queries regardless of the number of repo_ids — one per |
| 304 | signal. All queries use ``= ANY(:ids)`` for batching. |
| 305 | |
| 306 | Returns a dict mapping repo_id → RepoCardEnrichment. Repos with no intel |
| 307 | data get a safe zero-value enrichment so the caller never needs to guard |
| 308 | against missing keys. |
| 309 | |
| 310 | Parameters |
| 311 | ---------- |
| 312 | db: async SQLAlchemy session |
| 313 | repo_ids: list of sha256-prefixed repo identifiers to enrich |
| 314 | |
| 315 | Returns |
| 316 | ------- |
| 317 | dict[str, RepoCardEnrichment] — one entry per requested repo_id |
| 318 | """ |
| 319 | if not repo_ids: |
| 320 | return {} |
| 321 | |
| 322 | today = datetime.now(tz=timezone.utc).date() |
| 323 | window_start = datetime.now(tz=timezone.utc) - timedelta(days=_PULSE_DAYS) |
| 324 | |
| 325 | # Initialise result map with zero-value enrichments |
| 326 | results: dict[str, RepoCardEnrichment] = { |
| 327 | rid: RepoCardEnrichment( |
| 328 | repo_id=rid, |
| 329 | pulse_buckets=_build_pulse({}, today), |
| 330 | ) |
| 331 | for rid in repo_ids |
| 332 | } |
| 333 | |
| 334 | # ── Query 1: pulse — daily commit counts ───────────────────────────────── |
| 335 | day_col = func.cast( |
| 336 | func.date_trunc("day", MusehubCommit.timestamp.op("AT TIME ZONE")("UTC")), |
| 337 | sa.Date, |
| 338 | ).label("day") |
| 339 | pulse_rows = (await db.execute( |
| 340 | select( |
| 341 | MusehubCommitRef.repo_id, |
| 342 | day_col, |
| 343 | func.count().label("cnt"), |
| 344 | ) |
| 345 | .join(MusehubCommit, MusehubCommit.commit_id == MusehubCommitRef.commit_id) |
| 346 | .where( |
| 347 | MusehubCommitRef.repo_id.in_(repo_ids), |
| 348 | MusehubCommit.timestamp >= window_start, |
| 349 | ) |
| 350 | .group_by(MusehubCommitRef.repo_id, day_col) |
| 351 | )).fetchall() |
| 352 | |
| 353 | pulse_raw: dict[str, IntDict] = {rid: {} for rid in repo_ids} |
| 354 | for row in pulse_rows: |
| 355 | pulse_raw[row.repo_id][row.day.isoformat()] = row.cnt |
| 356 | for rid, raw in pulse_raw.items(): |
| 357 | results[rid].pulse_buckets = _build_pulse(raw, today) |
| 358 | |
| 359 | # ── Query 2: autonomy — agent vs human commit ratio ─────────────────────── |
| 360 | is_agent = case( |
| 361 | ( |
| 362 | (MusehubCommit.agent_id.isnot(None)) & (MusehubCommit.agent_id != ""), |
| 363 | 1, |
| 364 | ), |
| 365 | else_=0, |
| 366 | ) |
| 367 | autonomy_rows = (await db.execute( |
| 368 | select( |
| 369 | MusehubCommitRef.repo_id, |
| 370 | func.sum(is_agent).label("agent_commits"), |
| 371 | func.count().label("total_commits"), |
| 372 | ) |
| 373 | .join(MusehubCommit, MusehubCommit.commit_id == MusehubCommitRef.commit_id) |
| 374 | .where(MusehubCommitRef.repo_id.in_(repo_ids)) |
| 375 | .group_by(MusehubCommitRef.repo_id) |
| 376 | )).fetchall() |
| 377 | for row in autonomy_rows: |
| 378 | if row.total_commits > 0: |
| 379 | results[row.repo_id].autonomy_pct = round( |
| 380 | (row.agent_commits / row.total_commits) * 100 |
| 381 | ) |
| 382 | |
| 383 | # ── Query 3: hottest symbol — highest churn_30d per repo ───────────────── |
| 384 | # Use row_number() to pick the top symbol per repo without DISTINCT ON. |
| 385 | hottest_rn = ( |
| 386 | select( |
| 387 | MusehubSymbolIntel.repo_id, |
| 388 | MusehubSymbolIntel.address, |
| 389 | MusehubSymbolIntel.churn_30d, |
| 390 | MusehubSymbolIntel.blast, |
| 391 | func.row_number().over( |
| 392 | partition_by=MusehubSymbolIntel.repo_id, |
| 393 | order_by=MusehubSymbolIntel.churn_30d.desc().nulls_last(), |
| 394 | ).label("rn"), |
| 395 | ) |
| 396 | .where( |
| 397 | MusehubSymbolIntel.repo_id.in_(repo_ids), |
| 398 | MusehubSymbolIntel.churn_30d > 0, |
| 399 | ) |
| 400 | .subquery() |
| 401 | ) |
| 402 | hottest_rows = (await db.execute( |
| 403 | select(hottest_rn).where(hottest_rn.c.rn == 1) |
| 404 | )).fetchall() |
| 405 | for row in hottest_rows: |
| 406 | results[row.repo_id].hottest_symbol = SymbolStat( |
| 407 | address=row.address, |
| 408 | churn_30d=row.churn_30d, |
| 409 | blast=row.blast, |
| 410 | ) |
| 411 | |
| 412 | # ── Query 4: blast leader — highest blast score per repo ───────────────── |
| 413 | blast_rn = ( |
| 414 | select( |
| 415 | MusehubSymbolIntel.repo_id, |
| 416 | MusehubSymbolIntel.address, |
| 417 | MusehubSymbolIntel.churn_30d, |
| 418 | MusehubSymbolIntel.blast, |
| 419 | func.row_number().over( |
| 420 | partition_by=MusehubSymbolIntel.repo_id, |
| 421 | order_by=MusehubSymbolIntel.blast.desc().nulls_last(), |
| 422 | ).label("rn"), |
| 423 | ) |
| 424 | .where( |
| 425 | MusehubSymbolIntel.repo_id.in_(repo_ids), |
| 426 | MusehubSymbolIntel.blast > 0, |
| 427 | ) |
| 428 | .subquery() |
| 429 | ) |
| 430 | blast_rows = (await db.execute( |
| 431 | select(blast_rn).where(blast_rn.c.rn == 1) |
| 432 | )).fetchall() |
| 433 | for row in blast_rows: |
| 434 | results[row.repo_id].blast_leader = SymbolStat( |
| 435 | address=row.address, |
| 436 | churn_30d=row.churn_30d, |
| 437 | blast=row.blast, |
| 438 | ) |
| 439 | |
| 440 | # ── Query 5: health — dead count + breakage errors/warnings ────────────── |
| 441 | dead_rows = (await db.execute( |
| 442 | select( |
| 443 | MusehubIntelDead.repo_id, |
| 444 | func.count().label("dead_count"), |
| 445 | ) |
| 446 | .where( |
| 447 | MusehubIntelDead.repo_id.in_(repo_ids), |
| 448 | MusehubIntelDead.confidence == "high", |
| 449 | ) |
| 450 | .group_by(MusehubIntelDead.repo_id) |
| 451 | )).fetchall() |
| 452 | for row in dead_rows: |
| 453 | results[row.repo_id].dead_count = row.dead_count |
| 454 | |
| 455 | breakage_rows = (await db.execute( |
| 456 | select( |
| 457 | MusehubIntelBreakageMeta.repo_id, |
| 458 | MusehubIntelBreakageMeta.error_count, |
| 459 | MusehubIntelBreakageMeta.warning_count, |
| 460 | ) |
| 461 | .where(MusehubIntelBreakageMeta.repo_id.in_(repo_ids)) |
| 462 | )).fetchall() |
| 463 | for row in breakage_rows: |
| 464 | results[row.repo_id].error_count = row.error_count |
| 465 | results[row.repo_id].warning_count = row.warning_count |
| 466 | |
| 467 | return results |
File History
12 commits
sha256:73f24973678ceefd56628ee17c6d0535d86e33533828af6c63348f50941515ad
Merge branch 'fix/smoke-test-tag-label' into dev
Human
16 days ago
sha256:d035733f21ccff27735fddebfbbe0ed24565a32a22db8de5885402262671ecd2
chore: bump version to 0.2.0rc15 for musehub#113 fix release
Sonnet 4.6
patch
16 days ago
sha256:0032d6cfa33bc3c8367436ad768e7dd0e339b4332153160247da8266cb5fa352
Merge branch 'task/version-tags-phase3-server' into dev
Human
18 days ago
sha256:4669620efda9ff41c55bdefd1f7bfe1c239d468428744c84ead9957e5a003a53
merge: rescue snapshot-recovery hardening (c00aa21d) into d…
Opus 4.8
minor
⚠
31 days ago
sha256:a59da49c4611b970fc4b6ae48678ce4943261c213a07ddbd73ce9201df869b4a
fix: remove false-positive proposal_comments index drop fro…
Sonnet 4.6
patch
35 days ago
sha256:0a240d6dbff234f07d98a28a4a9a68db702f3f9ff9260196f24219bdb1c0b6f3
feat: render markdown mists as HTML with heading anchor links
Sonnet 4.6
patch
36 days ago
sha256:24a7d47486ebc4ebd1832830580e177ec6f877b48dced8c000e198cdec4ce9d6
Merge 'task/bump-version-rc12' into 'dev' — proposal: Bump …
Human
37 days ago
sha256:b9ff931d147e0114a1f17060f415b89ed551c170a91ff226c70437aa5c85f9ee
Merge 'task/bump-version-rc12' into 'dev' — proposal: Bump …
Human
37 days ago
sha256:d1122d21e73471879b460037b22c0b50fded7c423444a176f248428f75dac39c
Merge 'task/fix-issue-pagination-cursor' into 'dev' — propo…
Human
37 days ago
sha256:01e18975e73d2b3cd5b6db7929c895bef9aa6e0d4391dc5b2adfc548b41318dd
Merge 'feat/adding-debug-logs-to-staging' into 'dev' — prop…
Human
37 days ago
sha256:6b1949fc2797ca4c1936a637a4cbfec828ef56cf52398a2e74ca3c4f494e728f
fix: use wire_bytes not mpack_bytes_raw in compute_object_b…
Sonnet 4.6
patch
49 days ago
sha256:b99f2455dc346966d040133f5203297e6e3ef5803a93728a2c30568d0a0f7583
rename: delta_add → delta_upsert across wire format, models…
Sonnet 4.6
patch
51 days ago