gabriel / musehub public
musehub_search.py python
325 lines 10.8 KB
Raw
sha256:969ddc5e88776e70af33c016b8777bb721356508ae5f304b3fb46c451494ece7 test(mwp3): Phase 4 — fix adjacent suite regressions, lock … Sonnet 4.6 22 days ago
1 """MuseHub in-repo search service.
2
3 Provides four search modes over a repo's commit history, all operating on the
4 shared ``muse_cli_commits`` table and scoped to a single ``repo_id``.
5
6 Modes and their underlying algorithms:
7 - ``property`` — musical property filter (delegates to :mod:`musehub.services.muse_find`)
8 - ``ask`` — natural-language query; keyword extraction + overlap scoring
9 - ``keyword`` — raw keyword/phrase overlap (normalised overlap coefficient)
10 - ``pattern`` — substring pattern match against message and branch name
11
12 All four modes return :class:`~musehub.models.musehub.SearchResponse` so the
13 UI can render results with a single shared commit-row template regardless of mode.
14
15 Date-range filtering (``since`` / ``until``) is applied at the SQL layer for
16 efficiency before any Python-level scoring.
17 """
18
19 import logging
20 import re
21 from datetime import datetime
22
23 from sqlalchemy import and_
24 from sqlalchemy.ext.asyncio import AsyncSession
25 from sqlalchemy.future import select
26
27 from musehub.models.musehub import SearchCommitMatch, SearchResponse
28 from musehub.muse_cli.models import MuseCliCommit
29
30 logger = logging.getLogger(__name__)
31
32 _DEFAULT_LIMIT = 20
33 _TOKEN_RE = re.compile(r"[a-zA-Z0-9]+")
34
35 # Stop-words stripped during NL ask-mode keyword extraction.
36 _STOP_WORDS = frozenset({
37 "a", "an", "the", "is", "are", "was", "were", "be", "been", "being",
38 "have", "has", "had", "do", "does", "did", "will", "would", "could",
39 "should", "may", "might", "shall", "can", "need", "dare", "ought",
40 "i", "my", "me", "we", "our", "you", "your", "he", "she", "it",
41 "they", "their", "them", "what", "when", "where", "who", "which",
42 "how", "why", "in", "on", "at", "to", "of", "for", "and", "or",
43 "but", "not", "with", "from", "by", "about", "into", "through",
44 "did", "make", "made", "last", "any", "all", "that", "this",
45 })
46
47
48 # ---------------------------------------------------------------------------
49 # Internal helpers
50 # ---------------------------------------------------------------------------
51
52
53 def _tokenize(text: str) -> set[str]:
54 """Return a set of lowercase word tokens from *text*."""
55 return {m.group().lower() for m in _TOKEN_RE.finditer(text)}
56
57
58 def _overlap_score(query_tokens: set[str], message: str) -> float:
59 """Normalised overlap coefficient: |Q ∩ M| / |Q|.
60
61 Returns 1.0 when every query token appears in the message, 0.0 when
62 none do. Returns 0.0 for an empty query set to avoid division by zero.
63 """
64 if not query_tokens:
65 return 0.0
66 message_tokens = _tokenize(message)
67 return len(query_tokens & message_tokens) / len(query_tokens)
68
69
70 async def _fetch_candidates(
71 session: AsyncSession,
72 *,
73 repo_id: str,
74 since: datetime | None,
75 until: datetime | None,
76 cap: int = 5000,
77 ) -> tuple[list[MuseCliCommit], int]:
78 """Fetch candidate commits from DB with optional date range filter.
79
80 Returns ``(rows, total_scanned)`` where ``total_scanned`` is the raw DB count
81 before any Python-level filtering. We over-fetch (up to ``cap``) and let
82 callers apply their own ranking/limit so the SQL stays simple and fast.
83 """
84 stmt = select(MuseCliCommit).where(MuseCliCommit.repo_id == repo_id)
85
86 conditions = []
87 if since is not None:
88 conditions.append(MuseCliCommit.committed_at >= since)
89 if until is not None:
90 conditions.append(MuseCliCommit.committed_at <= until)
91 if conditions:
92 stmt = stmt.where(and_(*conditions))
93
94 stmt = stmt.order_by(MuseCliCommit.committed_at.desc()).limit(cap)
95
96 result = await session.execute(stmt)
97 rows = list(result.scalars().all())
98 return rows, len(rows)
99
100
101 def _commit_to_match(
102 commit: MuseCliCommit,
103 *,
104 score: float = 1.0,
105 match_source: str = "message",
106 ) -> SearchCommitMatch:
107 """Convert a DB row to the wire-format :class:`SearchCommitMatch`."""
108 return SearchCommitMatch(
109 commit_id=commit.commit_id,
110 branch=commit.branch,
111 message=commit.message,
112 author=commit.author,
113 timestamp=commit.committed_at,
114 score=round(score, 4),
115 match_source=match_source,
116 )
117
118
119 # ---------------------------------------------------------------------------
120 # Search modes
121 # ---------------------------------------------------------------------------
122
123
124 async def search_by_property(
125 session: AsyncSession,
126 *,
127 repo_id: str,
128 harmony: str | None = None,
129 rhythm: str | None = None,
130 melody: str | None = None,
131 structure: str | None = None,
132 dynamic: str | None = None,
133 emotion: str | None = None,
134 since: datetime | None = None,
135 until: datetime | None = None,
136 limit: int = _DEFAULT_LIMIT,
137 ) -> SearchResponse:
138 """Musical property filter. Currently returns an empty result set."""
139 active_filters = {
140 k: v for k, v in {
141 "harmony": harmony,
142 "rhythm": rhythm,
143 "melody": melody,
144 "structure": structure,
145 "dynamic": dynamic,
146 "emotion": emotion,
147 }.items() if v is not None
148 }
149 query_echo = " AND ".join(f"{k}={v}" for k, v in active_filters.items()) or "(all commits)"
150 logger.warning("⚠️ musehub search property: muse_find not available (muse-extraction)")
151 return SearchResponse(
152 mode="property",
153 query=query_echo,
154 matches=[],
155 total_scanned=0,
156 limit=limit,
157 )
158
159
160 async def search_by_ask(
161 session: AsyncSession,
162 *,
163 repo_id: str,
164 question: str,
165 since: datetime | None = None,
166 until: datetime | None = None,
167 limit: int = _DEFAULT_LIMIT,
168 ) -> SearchResponse:
169 """Natural-language query — keyword extraction + overlap scoring.
170
171 Strips stop-words from the question to produce a focused keyword set,
172 then ranks commits by overlap coefficient. Commits with zero overlap
173 are excluded. Returns at most ``limit`` results ordered by score desc.
174
175 This is a stub implementation; LLM-powered answer generation is a planned
176 enhancement that will replace the keyword scoring step.
177
178 Args:
179 session: Async SQLAlchemy session.
180 repo_id: Repo to search.
181 question: Natural-language question string.
182 since: Earliest committed_at (inclusive).
183 until: Latest committed_at (inclusive).
184 limit: Maximum results to return.
185
186 Returns:
187 :class:`~musehub.models.musehub.SearchResponse` with mode="ask".
188 """
189 rows, total_scanned = await _fetch_candidates(
190 session, repo_id=repo_id, since=since, until=until
191 )
192
193 # Extract meaningful keywords after stop-word removal.
194 tokens_raw = re.split(r"[\s\W]+", question.lower())
195 keywords: set[str] = {t for t in tokens_raw if t and t not in _STOP_WORDS and len(t) > 1}
196
197 scored: list[tuple[float, MuseCliCommit]] = []
198 for commit in rows:
199 if keywords:
200 score = _overlap_score(keywords, commit.message)
201 else:
202 # No useful tokens → include all commits with neutral score.
203 score = 1.0
204 if score > 0.0:
205 scored.append((score, commit))
206
207 scored.sort(key=lambda x: (x[0], x[1].committed_at.timestamp()), reverse=True)
208 top = scored[:limit]
209
210 matches = [_commit_to_match(c, score=s, match_source="message") for s, c in top]
211
212 logger.info("✅ musehub search ask: %d matches (repo=%s)", len(matches), repo_id)
213 return SearchResponse(
214 mode="ask",
215 query=question,
216 matches=matches,
217 total_scanned=total_scanned,
218 limit=limit,
219 )
220
221
222 async def search_by_keyword(
223 session: AsyncSession,
224 *,
225 repo_id: str,
226 keyword: str,
227 threshold: float = 0.0,
228 since: datetime | None = None,
229 until: datetime | None = None,
230 limit: int = _DEFAULT_LIMIT,
231 ) -> SearchResponse:
232 """Keyword search — overlap coefficient over commit messages.
233
234 Tokenises both *keyword* and each commit message, then scores using the
235 overlap coefficient. Commits below *threshold* are excluded.
236
237 Args:
238 session: Async SQLAlchemy session.
239 repo_id: Repo to search.
240 keyword: Keyword or phrase to search for.
241 threshold: Minimum overlap score [0, 1] to include a commit (default 0 = any match).
242 since: Earliest committed_at (inclusive).
243 until: Latest committed_at (inclusive).
244 limit: Maximum results to return.
245
246 Returns:
247 :class:`~musehub.models.musehub.SearchResponse` with mode="keyword".
248 """
249 rows, total_scanned = await _fetch_candidates(
250 session, repo_id=repo_id, since=since, until=until
251 )
252
253 query_tokens = _tokenize(keyword)
254 scored: list[tuple[float, MuseCliCommit]] = []
255 for commit in rows:
256 score = _overlap_score(query_tokens, commit.message)
257 if score >= threshold and score > 0.0:
258 scored.append((score, commit))
259
260 scored.sort(key=lambda x: (x[0], x[1].committed_at.timestamp()), reverse=True)
261 top = scored[:limit]
262
263 matches = [_commit_to_match(c, score=s, match_source="message") for s, c in top]
264
265 logger.info("✅ musehub search keyword: %d matches (repo=%s)", len(matches), repo_id)
266 return SearchResponse(
267 mode="keyword",
268 query=keyword,
269 matches=matches,
270 total_scanned=total_scanned,
271 limit=limit,
272 )
273
274
275 async def search_by_pattern(
276 session: AsyncSession,
277 *,
278 repo_id: str,
279 pattern: str,
280 since: datetime | None = None,
281 until: datetime | None = None,
282 limit: int = _DEFAULT_LIMIT,
283 ) -> SearchResponse:
284 """Pattern search — case-insensitive substring match against message and branch.
285
286 Matches commits where *pattern* appears anywhere in the commit message or
287 the branch name. Prioritises message matches over branch-name matches in
288 the result ordering.
289
290 Args:
291 session: Async SQLAlchemy session.
292 repo_id: Repo to search.
293 pattern: Substring pattern to search for.
294 since: Earliest committed_at (inclusive).
295 until: Latest committed_at (inclusive).
296 limit: Maximum results to return.
297
298 Returns:
299 :class:`~musehub.models.musehub.SearchResponse` with mode="pattern".
300 """
301 rows, total_scanned = await _fetch_candidates(
302 session, repo_id=repo_id, since=since, until=until
303 )
304
305 pat = pattern.lower()
306 message_matches: list[SearchCommitMatch] = []
307 branch_matches: list[SearchCommitMatch] = []
308
309 for commit in rows:
310 if pat in commit.message.lower():
311 message_matches.append(_commit_to_match(commit, match_source="message"))
312 elif pat in commit.branch.lower():
313 branch_matches.append(_commit_to_match(commit, match_source="branch"))
314
315 # Message matches come first, then branch matches.
316 all_matches = (message_matches + branch_matches)[:limit]
317
318 logger.info("✅ musehub search pattern: %d matches (repo=%s)", len(all_matches), repo_id)
319 return SearchResponse(
320 mode="pattern",
321 query=pattern,
322 matches=all_matches,
323 total_scanned=total_scanned,
324 limit=limit,
325 )
File History 2 commits
sha256:969ddc5e88776e70af33c016b8777bb721356508ae5f304b3fb46c451494ece7 test(mwp3): Phase 4 — fix adjacent suite regressions, lock … Sonnet 4.6 22 days ago
sha256:1749b9cc5cd2583c56d3261c4c00a342c27435f3946d4bc3e70f76aa2795458a docs(mwp-3): TDD implementation plan for job-enqueue idempo… Opus 4.8 22 days ago