musehub_search.py
python
sha256:8a1389ae62d0a688d5e027763249bd8faedfc39ab00857d65da6620d1ab8a689
Merge 'feat/9a-4-f7-overseer-provenance' into 'dev' — propo…
Human
4 days ago
| 1 | """MuseHub in-repo search service. |
| 2 | |
| 3 | Provides four search modes over a repo's commit history, all operating on the |
| 4 | shared ``muse_cli_commits`` table and scoped to a single ``repo_id``. |
| 5 | |
| 6 | Modes and their underlying algorithms: |
| 7 | - ``property`` — musical property filter (delegates to :mod:`musehub.services.muse_find`) |
| 8 | - ``ask`` — natural-language query; keyword extraction + overlap scoring |
| 9 | - ``keyword`` — raw keyword/phrase overlap (normalised overlap coefficient) |
| 10 | - ``pattern`` — substring pattern match against message and branch name |
| 11 | |
| 12 | All four modes return :class:`~musehub.models.musehub.SearchResponse` so the |
| 13 | UI can render results with a single shared commit-row template regardless of mode. |
| 14 | |
| 15 | Date-range filtering (``since`` / ``until``) is applied at the SQL layer for |
| 16 | efficiency before any Python-level scoring. |
| 17 | """ |
| 18 | |
| 19 | import logging |
| 20 | import re |
| 21 | from datetime import datetime |
| 22 | |
| 23 | from sqlalchemy import and_ |
| 24 | from sqlalchemy.ext.asyncio import AsyncSession |
| 25 | from sqlalchemy.future import select |
| 26 | |
| 27 | from musehub.models.musehub import SearchCommitMatch, SearchResponse |
| 28 | from musehub.muse_cli.models import MuseCliCommit |
| 29 | |
| 30 | logger = logging.getLogger(__name__) |
| 31 | |
| 32 | _DEFAULT_LIMIT = 20 |
| 33 | _TOKEN_RE = re.compile(r"[a-zA-Z0-9]+") |
| 34 | |
| 35 | # Stop-words stripped during NL ask-mode keyword extraction. |
| 36 | _STOP_WORDS = frozenset({ |
| 37 | "a", "an", "the", "is", "are", "was", "were", "be", "been", "being", |
| 38 | "have", "has", "had", "do", "does", "did", "will", "would", "could", |
| 39 | "should", "may", "might", "shall", "can", "need", "dare", "ought", |
| 40 | "i", "my", "me", "we", "our", "you", "your", "he", "she", "it", |
| 41 | "they", "their", "them", "what", "when", "where", "who", "which", |
| 42 | "how", "why", "in", "on", "at", "to", "of", "for", "and", "or", |
| 43 | "but", "not", "with", "from", "by", "about", "into", "through", |
| 44 | "did", "make", "made", "last", "any", "all", "that", "this", |
| 45 | }) |
| 46 | |
| 47 | |
| 48 | # --------------------------------------------------------------------------- |
| 49 | # Internal helpers |
| 50 | # --------------------------------------------------------------------------- |
| 51 | |
| 52 | |
| 53 | def _tokenize(text: str) -> set[str]: |
| 54 | """Return a set of lowercase word tokens from *text*.""" |
| 55 | return {m.group().lower() for m in _TOKEN_RE.finditer(text)} |
| 56 | |
| 57 | |
| 58 | def _overlap_score(query_tokens: set[str], message: str) -> float: |
| 59 | """Normalised overlap coefficient: |Q ∩ M| / |Q|. |
| 60 | |
| 61 | Returns 1.0 when every query token appears in the message, 0.0 when |
| 62 | none do. Returns 0.0 for an empty query set to avoid division by zero. |
| 63 | """ |
| 64 | if not query_tokens: |
| 65 | return 0.0 |
| 66 | message_tokens = _tokenize(message) |
| 67 | return len(query_tokens & message_tokens) / len(query_tokens) |
| 68 | |
| 69 | |
| 70 | async def _fetch_candidates( |
| 71 | session: AsyncSession, |
| 72 | *, |
| 73 | repo_id: str, |
| 74 | since: datetime | None, |
| 75 | until: datetime | None, |
| 76 | cap: int = 5000, |
| 77 | ) -> tuple[list[MuseCliCommit], int]: |
| 78 | """Fetch candidate commits from DB with optional date range filter. |
| 79 | |
| 80 | Returns ``(rows, total_scanned)`` where ``total_scanned`` is the raw DB count |
| 81 | before any Python-level filtering. We over-fetch (up to ``cap``) and let |
| 82 | callers apply their own ranking/limit so the SQL stays simple and fast. |
| 83 | """ |
| 84 | stmt = select(MuseCliCommit).where(MuseCliCommit.repo_id == repo_id) |
| 85 | |
| 86 | conditions = [] |
| 87 | if since is not None: |
| 88 | conditions.append(MuseCliCommit.committed_at >= since) |
| 89 | if until is not None: |
| 90 | conditions.append(MuseCliCommit.committed_at <= until) |
| 91 | if conditions: |
| 92 | stmt = stmt.where(and_(*conditions)) |
| 93 | |
| 94 | stmt = stmt.order_by(MuseCliCommit.committed_at.desc()).limit(cap) |
| 95 | |
| 96 | result = await session.execute(stmt) |
| 97 | rows = list(result.scalars().all()) |
| 98 | return rows, len(rows) |
| 99 | |
| 100 | |
| 101 | def _commit_to_match( |
| 102 | commit: MuseCliCommit, |
| 103 | *, |
| 104 | score: float = 1.0, |
| 105 | match_source: str = "message", |
| 106 | ) -> SearchCommitMatch: |
| 107 | """Convert a DB row to the wire-format :class:`SearchCommitMatch`.""" |
| 108 | return SearchCommitMatch( |
| 109 | commit_id=commit.commit_id, |
| 110 | branch=commit.branch, |
| 111 | message=commit.message, |
| 112 | author=commit.author, |
| 113 | timestamp=commit.committed_at, |
| 114 | score=round(score, 4), |
| 115 | match_source=match_source, |
| 116 | ) |
| 117 | |
| 118 | |
| 119 | # --------------------------------------------------------------------------- |
| 120 | # Search modes |
| 121 | # --------------------------------------------------------------------------- |
| 122 | |
| 123 | |
| 124 | async def search_by_property( |
| 125 | session: AsyncSession, |
| 126 | *, |
| 127 | repo_id: str, |
| 128 | harmony: str | None = None, |
| 129 | rhythm: str | None = None, |
| 130 | melody: str | None = None, |
| 131 | structure: str | None = None, |
| 132 | dynamic: str | None = None, |
| 133 | emotion: str | None = None, |
| 134 | since: datetime | None = None, |
| 135 | until: datetime | None = None, |
| 136 | limit: int = _DEFAULT_LIMIT, |
| 137 | ) -> SearchResponse: |
| 138 | """Musical property filter. Currently returns an empty result set.""" |
| 139 | active_filters = { |
| 140 | k: v for k, v in { |
| 141 | "harmony": harmony, |
| 142 | "rhythm": rhythm, |
| 143 | "melody": melody, |
| 144 | "structure": structure, |
| 145 | "dynamic": dynamic, |
| 146 | "emotion": emotion, |
| 147 | }.items() if v is not None |
| 148 | } |
| 149 | query_echo = " AND ".join(f"{k}={v}" for k, v in active_filters.items()) or "(all commits)" |
| 150 | logger.warning("⚠️ musehub search property: muse_find not available (muse-extraction)") |
| 151 | return SearchResponse( |
| 152 | mode="property", |
| 153 | query=query_echo, |
| 154 | matches=[], |
| 155 | total_scanned=0, |
| 156 | limit=limit, |
| 157 | ) |
| 158 | |
| 159 | |
| 160 | async def search_by_ask( |
| 161 | session: AsyncSession, |
| 162 | *, |
| 163 | repo_id: str, |
| 164 | question: str, |
| 165 | since: datetime | None = None, |
| 166 | until: datetime | None = None, |
| 167 | limit: int = _DEFAULT_LIMIT, |
| 168 | ) -> SearchResponse: |
| 169 | """Natural-language query — keyword extraction + overlap scoring. |
| 170 | |
| 171 | Strips stop-words from the question to produce a focused keyword set, |
| 172 | then ranks commits by overlap coefficient. Commits with zero overlap |
| 173 | are excluded. Returns at most ``limit`` results ordered by score desc. |
| 174 | |
| 175 | This is a stub implementation; LLM-powered answer generation is a planned |
| 176 | enhancement that will replace the keyword scoring step. |
| 177 | |
| 178 | Args: |
| 179 | session: Async SQLAlchemy session. |
| 180 | repo_id: Repo to search. |
| 181 | question: Natural-language question string. |
| 182 | since: Earliest committed_at (inclusive). |
| 183 | until: Latest committed_at (inclusive). |
| 184 | limit: Maximum results to return. |
| 185 | |
| 186 | Returns: |
| 187 | :class:`~musehub.models.musehub.SearchResponse` with mode="ask". |
| 188 | """ |
| 189 | rows, total_scanned = await _fetch_candidates( |
| 190 | session, repo_id=repo_id, since=since, until=until |
| 191 | ) |
| 192 | |
| 193 | # Extract meaningful keywords after stop-word removal. |
| 194 | tokens_raw = re.split(r"[\s\W]+", question.lower()) |
| 195 | keywords: set[str] = {t for t in tokens_raw if t and t not in _STOP_WORDS and len(t) > 1} |
| 196 | |
| 197 | scored: list[tuple[float, MuseCliCommit]] = [] |
| 198 | for commit in rows: |
| 199 | if keywords: |
| 200 | score = _overlap_score(keywords, commit.message) |
| 201 | else: |
| 202 | # No useful tokens → include all commits with neutral score. |
| 203 | score = 1.0 |
| 204 | if score > 0.0: |
| 205 | scored.append((score, commit)) |
| 206 | |
| 207 | scored.sort(key=lambda x: (x[0], x[1].committed_at.timestamp()), reverse=True) |
| 208 | top = scored[:limit] |
| 209 | |
| 210 | matches = [_commit_to_match(c, score=s, match_source="message") for s, c in top] |
| 211 | |
| 212 | logger.info("✅ musehub search ask: %d matches (repo=%s)", len(matches), repo_id) |
| 213 | return SearchResponse( |
| 214 | mode="ask", |
| 215 | query=question, |
| 216 | matches=matches, |
| 217 | total_scanned=total_scanned, |
| 218 | limit=limit, |
| 219 | ) |
| 220 | |
| 221 | |
| 222 | async def search_by_keyword( |
| 223 | session: AsyncSession, |
| 224 | *, |
| 225 | repo_id: str, |
| 226 | keyword: str, |
| 227 | threshold: float = 0.0, |
| 228 | since: datetime | None = None, |
| 229 | until: datetime | None = None, |
| 230 | limit: int = _DEFAULT_LIMIT, |
| 231 | ) -> SearchResponse: |
| 232 | """Keyword search — overlap coefficient over commit messages. |
| 233 | |
| 234 | Tokenises both *keyword* and each commit message, then scores using the |
| 235 | overlap coefficient. Commits below *threshold* are excluded. |
| 236 | |
| 237 | Args: |
| 238 | session: Async SQLAlchemy session. |
| 239 | repo_id: Repo to search. |
| 240 | keyword: Keyword or phrase to search for. |
| 241 | threshold: Minimum overlap score [0, 1] to include a commit (default 0 = any match). |
| 242 | since: Earliest committed_at (inclusive). |
| 243 | until: Latest committed_at (inclusive). |
| 244 | limit: Maximum results to return. |
| 245 | |
| 246 | Returns: |
| 247 | :class:`~musehub.models.musehub.SearchResponse` with mode="keyword". |
| 248 | """ |
| 249 | rows, total_scanned = await _fetch_candidates( |
| 250 | session, repo_id=repo_id, since=since, until=until |
| 251 | ) |
| 252 | |
| 253 | query_tokens = _tokenize(keyword) |
| 254 | scored: list[tuple[float, MuseCliCommit]] = [] |
| 255 | for commit in rows: |
| 256 | score = _overlap_score(query_tokens, commit.message) |
| 257 | if score >= threshold and score > 0.0: |
| 258 | scored.append((score, commit)) |
| 259 | |
| 260 | scored.sort(key=lambda x: (x[0], x[1].committed_at.timestamp()), reverse=True) |
| 261 | top = scored[:limit] |
| 262 | |
| 263 | matches = [_commit_to_match(c, score=s, match_source="message") for s, c in top] |
| 264 | |
| 265 | logger.info("✅ musehub search keyword: %d matches (repo=%s)", len(matches), repo_id) |
| 266 | return SearchResponse( |
| 267 | mode="keyword", |
| 268 | query=keyword, |
| 269 | matches=matches, |
| 270 | total_scanned=total_scanned, |
| 271 | limit=limit, |
| 272 | ) |
| 273 | |
| 274 | |
| 275 | async def search_by_pattern( |
| 276 | session: AsyncSession, |
| 277 | *, |
| 278 | repo_id: str, |
| 279 | pattern: str, |
| 280 | since: datetime | None = None, |
| 281 | until: datetime | None = None, |
| 282 | limit: int = _DEFAULT_LIMIT, |
| 283 | ) -> SearchResponse: |
| 284 | """Pattern search — case-insensitive substring match against message and branch. |
| 285 | |
| 286 | Matches commits where *pattern* appears anywhere in the commit message or |
| 287 | the branch name. Prioritises message matches over branch-name matches in |
| 288 | the result ordering. |
| 289 | |
| 290 | Args: |
| 291 | session: Async SQLAlchemy session. |
| 292 | repo_id: Repo to search. |
| 293 | pattern: Substring pattern to search for. |
| 294 | since: Earliest committed_at (inclusive). |
| 295 | until: Latest committed_at (inclusive). |
| 296 | limit: Maximum results to return. |
| 297 | |
| 298 | Returns: |
| 299 | :class:`~musehub.models.musehub.SearchResponse` with mode="pattern". |
| 300 | """ |
| 301 | rows, total_scanned = await _fetch_candidates( |
| 302 | session, repo_id=repo_id, since=since, until=until |
| 303 | ) |
| 304 | |
| 305 | pat = pattern.lower() |
| 306 | message_matches: list[SearchCommitMatch] = [] |
| 307 | branch_matches: list[SearchCommitMatch] = [] |
| 308 | |
| 309 | for commit in rows: |
| 310 | if pat in commit.message.lower(): |
| 311 | message_matches.append(_commit_to_match(commit, match_source="message")) |
| 312 | elif pat in commit.branch.lower(): |
| 313 | branch_matches.append(_commit_to_match(commit, match_source="branch")) |
| 314 | |
| 315 | # Message matches come first, then branch matches. |
| 316 | all_matches = (message_matches + branch_matches)[:limit] |
| 317 | |
| 318 | logger.info("✅ musehub search pattern: %d matches (repo=%s)", len(all_matches), repo_id) |
| 319 | return SearchResponse( |
| 320 | mode="pattern", |
| 321 | query=pattern, |
| 322 | matches=all_matches, |
| 323 | total_scanned=total_scanned, |
| 324 | limit=limit, |
| 325 | ) |
File History
15 commits
sha256:8a1389ae62d0a688d5e027763249bd8faedfc39ab00857d65da6620d1ab8a689
Merge 'feat/9a-4-f7-overseer-provenance' into 'dev' — propo…
Human
4 days ago
sha256:bee12c5cbde2334f98421c6c209d768fa6b8004d6705c9ea798ce6c1651bc11f
Merge 'infra/database-phase3-4-cleanup' into 'dev' — propos…
Human
4 days ago
sha256:fc04e4cae9e1774d6a21b65c45daeed0e6787eb581d13aa1b03bfe9384a34226
Merge branch 'fix/two-column-scroll-layout' into dev
Human
67 days ago
sha256:408916fc5973ba59c6e4eebaa80ebdcc801c0a63205651e25009d11548f79454
chore: bump version to 0.2.0.dev2 — nightly.2, matching muse
Sonnet 4.6
patch
70 days ago
sha256:d035733f21ccff27735fddebfbbe0ed24565a32a22db8de5885402262671ecd2
chore: bump version to 0.2.0rc15 for musehub#113 fix release
Sonnet 4.6
patch
73 days ago
sha256:0032d6cfa33bc3c8367436ad768e7dd0e339b4332153160247da8266cb5fa352
Merge branch 'task/version-tags-phase3-server' into dev
Human
76 days ago
sha256:4669620efda9ff41c55bdefd1f7bfe1c239d468428744c84ead9957e5a003a53
merge: rescue snapshot-recovery hardening (c00aa21d) into d…
Opus 4.8
minor
⚠
88 days ago
sha256:a59da49c4611b970fc4b6ae48678ce4943261c213a07ddbd73ce9201df869b4a
fix: remove false-positive proposal_comments index drop fro…
Sonnet 4.6
patch
92 days ago
sha256:0a240d6dbff234f07d98a28a4a9a68db702f3f9ff9260196f24219bdb1c0b6f3
feat: render markdown mists as HTML with heading anchor links
Sonnet 4.6
patch
93 days ago
sha256:24a7d47486ebc4ebd1832830580e177ec6f877b48dced8c000e198cdec4ce9d6
Merge 'task/bump-version-rc12' into 'dev' — proposal: Bump …
Human
94 days ago
sha256:b9ff931d147e0114a1f17060f415b89ed551c170a91ff226c70437aa5c85f9ee
Merge 'task/bump-version-rc12' into 'dev' — proposal: Bump …
Human
94 days ago
sha256:d1122d21e73471879b460037b22c0b50fded7c423444a176f248428f75dac39c
Merge 'task/fix-issue-pagination-cursor' into 'dev' — propo…
Human
94 days ago
sha256:01e18975e73d2b3cd5b6db7929c895bef9aa6e0d4391dc5b2adfc548b41318dd
Merge 'feat/adding-debug-logs-to-staging' into 'dev' — prop…
Human
94 days ago
sha256:6b1949fc2797ca4c1936a637a4cbfec828ef56cf52398a2e74ca3c4f494e728f
fix: use wire_bytes not mpack_bytes_raw in compute_object_b…
Sonnet 4.6
patch
107 days ago
sha256:b99f2455dc346966d040133f5203297e6e3ef5803a93728a2c30568d0a0f7583
rename: delta_add → delta_upsert across wire format, models…
Sonnet 4.6
patch
109 days ago