aaronrene / gabriel-muse public
content_grep.py python
587 lines 20.2 KB
Raw
sha256:cf6265cea8c21d9228d90dec13ef6ec2dab5103d466db9cc4590681832de4bf8 docs(KD-STAGING): sync governance after KD-6b DONE Human 13 days ago
1 """``muse content-grep`` — full-text search across tracked files.
2
3 Searches the content of every tracked file for a pattern. By default the
4 search runs against the committed HEAD snapshot (reading from the object
5 store). Pass ``--working-tree`` to search the actual files on disk,
6 including uncommitted edits — essential for agents verifying their own
7 changes before committing.
8
9 ``--working-tree`` and ``--ref`` are mutually exclusive.
10
11 Binary files and non-UTF-8 files are silently skipped. Regex safety:
12 patterns are compiled with a 500-character length limit to prevent
13 catastrophic backtracking (ReDoS).
14
15 Performance (snapshot mode): object reads run in parallel using a bounded
16 ``ThreadPoolExecutor`` (``min(8, cpu_count())`` workers).
17
18 File filtering: ``--include`` and ``--exclude`` accept ``fnmatch``-style
19 glob patterns applied to relative file paths.
20
21 Usage::
22
23 muse content-grep "Cm7" # literal substring (HEAD)
24 muse content-grep "TODO" --working-tree # search working tree (disk)
25 muse content-grep "tempo:\\s+\\d+" # regex
26 muse content-grep "TODO" --ignore-case # case-insensitive
27 muse content-grep "chorus" --files-only # only file paths
28 muse content-grep "bass" --ref feat/audio # search a branch tip
29 muse content-grep "note" --include "*.txt" # only .txt files
30 muse content-grep "debug" --exclude "*.min.js"
31 muse content-grep "TODO" --max-matches 20 # cap results
32 muse content-grep "verse" --context 2 # 2 lines of context
33 muse content-grep "chord" --json # machine-readable
34
35 JSON output schema (``--json``)::
36
37 {
38 "source": "commit" | "working-tree",
39 "commit_id": "<64-char hex>" | null,
40 "snapshot_id": "<64-char hex>" | null,
41 "pattern": "<pattern string>",
42 "total_files_matched": <int>,
43 "total_matches": <int>,
44 "results": [
45 {
46 "path": "<relative path>",
47 "object_id": "<64-char hex>" | null,
48 "match_count": <int>,
49 "matches": [
50 {
51 "line_number": <int>,
52 "text": "<matched line>",
53 "context_before": ["<line>", ...],
54 "context_after": ["<line>", ...]
55 },
56 ...
57 ]
58 },
59 ...
60 ]
61 }
62
63 Exit codes::
64
65 0 — pattern found in at least one file
66 1 — no matches (or no commits)
67 3 — I/O error
68 """
69
70 from __future__ import annotations
71
72 import argparse
73 import concurrent.futures
74 import fnmatch
75 import json
76 import logging
77 import os
78 import pathlib
79 import re
80 import sys
81 from typing import TypedDict
82
83 from muse.core.errors import ExitCode
84 from muse.core.object_store import read_object
85 from muse.core.repo import read_repo_id, require_repo
86 from muse.core.store import (
87 get_head_commit_id,
88 read_commit,
89 read_current_branch,
90 read_snapshot,
91 resolve_commit_ref,
92 )
93 from muse.core.validation import sanitize_display
94
95 logger = logging.getLogger(__name__)
96
97 _BINARY_CHUNK = 8192
98 _MAX_PATTERN_LEN = 500 # reject patterns that could cause catastrophic backtracking
99 _DEFAULT_MAX_WORKERS = min(8, (os.cpu_count() or 1))
100
101 # Directories to skip when walking the working tree.
102 _SKIP_DIRS: frozenset[str] = frozenset({
103 ".muse",
104 ".git",
105 "__pycache__",
106 ".mypy_cache",
107 ".pytest_cache",
108 ".tox",
109 "node_modules",
110 ".venv",
111 "venv",
112 ".env",
113 })
114
115
116 # ---------------------------------------------------------------------------
117 # TypedDicts for structured output
118 # ---------------------------------------------------------------------------
119
120
121 class GrepMatch(TypedDict):
122 """A single matching line within a file, with optional surrounding context."""
123
124 line_number: int
125 text: str
126 context_before: list[str]
127 context_after: list[str]
128
129
130 class GrepFileResult(TypedDict):
131 """All matches within a single file."""
132
133 path: str
134 object_id: str | None # None when source is working-tree
135 match_count: int
136 matches: list[GrepMatch]
137
138
139 class _ContentGrepJson(TypedDict):
140 """Top-level JSON envelope for ``muse content-grep --json``."""
141
142 source: str # "commit" | "working-tree"
143 commit_id: str | None
144 snapshot_id: str | None
145 pattern: str
146 total_files_matched: int
147 total_matches: int
148 results: list[GrepFileResult]
149
150
151 # ---------------------------------------------------------------------------
152 # Internal helpers
153 # ---------------------------------------------------------------------------
154
155
156 def _is_binary(data: bytes) -> bool:
157 """Return ``True`` if *data* (the first chunk) contains null bytes."""
158 return b"\x00" in data
159
160
161 def _path_matches_globs(rel_path: str, include: str | None, exclude: str | None) -> bool:
162 """Return ``True`` if *rel_path* passes the include/exclude glob filters.
163
164 ``--include`` and ``--exclude`` use ``fnmatch`` on the basename **and** on
165 the full relative path so that patterns like ``*.py`` and ``src/*.py`` both
166 work intuitively.
167 """
168 basename = pathlib.PurePosixPath(rel_path).name
169 if include is not None:
170 if not (fnmatch.fnmatch(basename, include) or fnmatch.fnmatch(rel_path, include)):
171 return False
172 if exclude is not None:
173 if fnmatch.fnmatch(basename, exclude) or fnmatch.fnmatch(rel_path, exclude):
174 return False
175 return True
176
177
178 def _search_lines(
179 raw: bytes,
180 pattern: re.Pattern[str],
181 files_only: bool,
182 count_only: bool,
183 context_lines: int,
184 ) -> tuple[int, list[GrepMatch]]:
185 """Search *raw* bytes for *pattern*; return ``(match_count, matches)``.
186
187 Binary content and non-UTF-8 content return ``(0, [])``.
188 """
189 probe = raw[:_BINARY_CHUNK]
190 if _is_binary(probe):
191 return 0, []
192
193 text = raw.decode("utf-8", errors="replace")
194 all_lines = text.splitlines()
195
196 matches: list[GrepMatch] = []
197 total = 0
198 for lineno, line in enumerate(all_lines, start=1):
199 if pattern.search(line):
200 total += 1
201 if not files_only and not count_only:
202 before: list[str] = []
203 after: list[str] = []
204 if context_lines > 0:
205 idx = lineno - 1 # 0-based index
206 before = [
207 l.rstrip("\r")
208 for l in all_lines[max(0, idx - context_lines) : idx]
209 ]
210 after = [
211 l.rstrip("\r")
212 for l in all_lines[idx + 1 : idx + 1 + context_lines]
213 ]
214 matches.append(
215 GrepMatch(
216 line_number=lineno,
217 text=line.rstrip("\r"),
218 context_before=before,
219 context_after=after,
220 )
221 )
222
223 return total, matches
224
225
226 def _search_object(
227 root_path: pathlib.Path,
228 object_id: str,
229 pattern: re.Pattern[str],
230 files_only: bool,
231 count_only: bool,
232 context_lines: int,
233 ) -> tuple[int, list[GrepMatch]]:
234 """Search a committed object for *pattern*; return ``(match_count, matches)``."""
235 try:
236 raw = read_object(root_path, object_id)
237 except OSError as exc:
238 logger.warning("⚠️ grep: could not read object %s: %s", object_id[:12], exc)
239 return 0, []
240
241 if raw is None:
242 return 0, []
243
244 return _search_lines(raw, pattern, files_only, count_only, context_lines)
245
246
247 def _search_disk_file(
248 abs_path: pathlib.Path,
249 pattern: re.Pattern[str],
250 files_only: bool,
251 count_only: bool,
252 context_lines: int,
253 ) -> tuple[int, list[GrepMatch]]:
254 """Search a file on disk for *pattern*; return ``(match_count, matches)``."""
255 try:
256 raw = abs_path.read_bytes()
257 except OSError as exc:
258 logger.warning("⚠️ grep: could not read %s: %s", abs_path, exc)
259 return 0, []
260
261 return _search_lines(raw, pattern, files_only, count_only, context_lines)
262
263
264 def _walk_working_tree(
265 root: pathlib.Path,
266 include_glob: str | None,
267 exclude_glob: str | None,
268 ) -> list[tuple[str, pathlib.Path]]:
269 """Walk *root* recursively, skipping VCS/cache dirs, return ``(rel_path, abs_path)`` pairs."""
270 results: list[tuple[str, pathlib.Path]] = []
271 for dirpath, dirnames, filenames in os.walk(root):
272 # Prune directories in-place so os.walk skips them entirely.
273 dirnames[:] = sorted(
274 d for d in dirnames
275 if d not in _SKIP_DIRS
276 and not d.startswith(".")
277 and not os.path.isdir(os.path.join(dirpath, d, ".muse"))
278 )
279 for filename in sorted(filenames):
280 abs_path = pathlib.Path(dirpath) / filename
281 try:
282 rel_path = abs_path.relative_to(root).as_posix()
283 except ValueError:
284 continue
285 if _path_matches_globs(rel_path, include_glob, exclude_glob):
286 results.append((rel_path, abs_path))
287 return results
288
289
290 # ---------------------------------------------------------------------------
291 # Registration
292 # ---------------------------------------------------------------------------
293
294
295 def register(subparsers: "argparse._SubParsersAction[argparse.ArgumentParser]") -> None:
296 """Register the content-grep subcommand."""
297 parser = subparsers.add_parser(
298 "content-grep",
299 help="Search tracked file content for a pattern.",
300 description=__doc__,
301 formatter_class=argparse.RawDescriptionHelpFormatter,
302 )
303 parser.add_argument(
304 "pattern",
305 help="Regular expression pattern to search for.",
306 )
307 parser.add_argument(
308 "--working-tree", "-w", action="store_true", dest="working_tree",
309 help=(
310 "Search files on disk (working tree) instead of the committed HEAD snapshot. "
311 "Finds matches in uncommitted edits. Mutually exclusive with --ref."
312 ),
313 )
314 parser.add_argument(
315 "--ref", default=None,
316 help="Branch, tag, or commit SHA to search (default: HEAD). Mutually exclusive with --working-tree.",
317 )
318 parser.add_argument(
319 "--ignore-case", "-i", action="store_true", dest="ignore_case",
320 help="Case-insensitive matching.",
321 )
322 parser.add_argument(
323 "--files-only", "-l", action="store_true", dest="files_only",
324 help="Print only file paths with matches.",
325 )
326 parser.add_argument(
327 "--count", "-c", action="store_true", dest="count_mode",
328 help="Print only match counts per file.",
329 )
330 parser.add_argument(
331 "--include", default=None, dest="include_glob",
332 help="Only search files whose path matches this fnmatch glob (e.g. '*.py').",
333 )
334 parser.add_argument(
335 "--exclude", default=None, dest="exclude_glob",
336 help="Skip files whose path matches this fnmatch glob (e.g. '*.min.js').",
337 )
338 parser.add_argument(
339 "--max-matches", "-m", type=int, default=None, dest="max_matches",
340 help="Stop after this many total matches across all files.",
341 )
342 parser.add_argument(
343 "--context", "-C", type=int, default=0, dest="context_lines",
344 help="Number of surrounding lines to include with each match (like grep -C).",
345 )
346 parser.add_argument(
347 "--json", action="store_true", dest="json_out",
348 help="Emit machine-readable JSON.",
349 )
350 parser.set_defaults(func=run)
351
352
353 # ---------------------------------------------------------------------------
354 # Subcommand handler
355 # ---------------------------------------------------------------------------
356
357
358 def run(args: argparse.Namespace) -> None:
359 """Search tracked file content for a pattern.
360
361 Pattern validation (length + ``re.compile``) is performed *before* any I/O
362 so that invalid patterns are rejected cheaply.
363
364 When ``--working-tree`` is set, files are read directly from disk so that
365 uncommitted edits are visible. This is the correct mode for agents
366 verifying changes before committing. Object reads (snapshot mode) are
367 parallelised with a ``ThreadPoolExecutor`` for I/O efficiency.
368
369 Binary files and non-UTF-8 files are silently skipped. Exit code 0 means
370 at least one match was found; exit code 1 means no matches.
371
372 Examples::
373
374 muse content-grep "chorus"
375 muse content-grep "TODO" --working-tree
376 muse content-grep "TODO|FIXME" --files-only
377 muse content-grep "tempo" --ignore-case --json
378 muse content-grep "chord" --ref feat/harmony
379 muse content-grep "note" --include "*.txt" --max-matches 10
380 muse content-grep "verse" --context 2
381 """
382 pattern: str = args.pattern
383 working_tree: bool = args.working_tree
384 ref: str | None = args.ref
385 ignore_case: bool = args.ignore_case
386 files_only: bool = args.files_only
387 count_mode: bool = args.count_mode
388 include_glob: str | None = args.include_glob
389 exclude_glob: str | None = args.exclude_glob
390 max_matches: int | None = args.max_matches
391 context_lines: int = max(0, args.context_lines)
392 json_out: bool = args.json_out
393
394 if working_tree and ref is not None:
395 print("❌ --working-tree and --ref are mutually exclusive.", file=sys.stderr)
396 raise SystemExit(ExitCode.USER_ERROR)
397
398 # Validate pattern BEFORE any I/O — cheap rejection of bad inputs.
399 if len(pattern) > _MAX_PATTERN_LEN:
400 print(
401 f"❌ Pattern too long ({len(pattern)} chars, max {_MAX_PATTERN_LEN}). "
402 "Use a shorter pattern or re.escape() for literal matches.",
403 file=sys.stderr,
404 )
405 raise SystemExit(ExitCode.USER_ERROR)
406
407 # Normalize BRE-style \| to ERE-style | so agents with grep muscle memory
408 # get the expected behaviour without needing to know the distinction.
409 pattern = pattern.replace(r"\|", "|")
410
411 flags = re.IGNORECASE if ignore_case else 0
412 try:
413 compiled: re.Pattern[str] = re.compile(pattern, flags)
414 except re.error as exc:
415 print(f"❌ Invalid regex: {exc}", file=sys.stderr)
416 raise SystemExit(ExitCode.USER_ERROR) from exc
417
418 root = require_repo()
419
420 # ── Working-tree mode ────────────────────────────────────────────────────
421 if working_tree:
422 disk_files = _walk_working_tree(root, include_glob, exclude_glob)
423
424 file_results: list[GrepFileResult] = []
425 total_matches = 0
426
427 for rel_path, abs_path in disk_files:
428 if max_matches is not None and total_matches >= max_matches:
429 break
430 match_count, matches = _search_disk_file(
431 abs_path, compiled, files_only, count_mode, context_lines
432 )
433 if match_count > 0:
434 if max_matches is not None:
435 remaining = max_matches - total_matches
436 if match_count > remaining:
437 matches = matches[:remaining]
438 match_count = len(matches)
439 file_results.append(
440 GrepFileResult(
441 path=rel_path,
442 object_id=None,
443 match_count=match_count,
444 matches=matches,
445 )
446 )
447 total_matches += match_count
448
449 _emit(
450 file_results=file_results,
451 total_matches=total_matches,
452 source="working-tree",
453 commit_id=None,
454 snapshot_id=None,
455 pattern=pattern,
456 json_out=json_out,
457 files_only=files_only,
458 count_mode=count_mode,
459 context_lines=context_lines,
460 )
461 if not file_results:
462 raise SystemExit(ExitCode.USER_ERROR) # exit 1 = no matches
463 return
464
465 # ── Snapshot mode (default) ──────────────────────────────────────────────
466 repo_id = read_repo_id(root)
467 branch = read_current_branch(root)
468
469 if ref is None:
470 commit_id = get_head_commit_id(root, branch)
471 if commit_id is None:
472 print("❌ No commits on current branch.", file=sys.stderr)
473 raise SystemExit(ExitCode.USER_ERROR)
474 else:
475 commit_rec = resolve_commit_ref(root, repo_id, branch, ref)
476 if commit_rec is None:
477 print(f"❌ Ref '{sanitize_display(ref)}' not found.", file=sys.stderr)
478 raise SystemExit(ExitCode.USER_ERROR)
479 commit_id = commit_rec.commit_id
480
481 commit = read_commit(root, commit_id)
482 if commit is None:
483 print(f"❌ Commit {commit_id[:12]} not found.", file=sys.stderr)
484 raise SystemExit(ExitCode.INTERNAL_ERROR)
485
486 snap = read_snapshot(root, commit.snapshot_id)
487 if snap is None:
488 print(f"❌ Snapshot {commit.snapshot_id[:12]} not found.", file=sys.stderr)
489 raise SystemExit(ExitCode.INTERNAL_ERROR)
490
491 filtered: list[tuple[str, str]] = [
492 (rel_path, object_id)
493 for rel_path, object_id in sorted(snap.manifest.items())
494 if _path_matches_globs(rel_path, include_glob, exclude_glob)
495 ]
496
497 def _search(item: tuple[str, str]) -> tuple[str, str, int, list[GrepMatch]]:
498 rel_path, object_id = item
499 cnt, ms = _search_object(root, object_id, compiled, files_only, count_mode, context_lines)
500 return rel_path, object_id, cnt, ms
501
502 snap_results: list[GrepFileResult] = []
503 snap_total = 0
504
505 with concurrent.futures.ThreadPoolExecutor(max_workers=_DEFAULT_MAX_WORKERS) as pool:
506 for rel_path, object_id, match_count, matches in pool.map(_search, filtered):
507 if match_count > 0:
508 if max_matches is not None:
509 remaining = max_matches - snap_total
510 if remaining <= 0:
511 break
512 if match_count > remaining:
513 matches = matches[:remaining]
514 match_count = len(matches)
515 snap_results.append(
516 GrepFileResult(
517 path=rel_path,
518 object_id=object_id,
519 match_count=match_count,
520 matches=matches,
521 )
522 )
523 snap_total += match_count
524
525 _emit(
526 file_results=snap_results,
527 total_matches=snap_total,
528 source="commit",
529 commit_id=commit_id,
530 snapshot_id=commit.snapshot_id,
531 pattern=pattern,
532 json_out=json_out,
533 files_only=files_only,
534 count_mode=count_mode,
535 context_lines=context_lines,
536 )
537 if not snap_results:
538 raise SystemExit(ExitCode.USER_ERROR) # exit 1 = no matches
539
540
541 # ---------------------------------------------------------------------------
542 # Output helper (shared between modes)
543 # ---------------------------------------------------------------------------
544
545
546 def _emit(
547 *,
548 file_results: list[GrepFileResult],
549 total_matches: int,
550 source: str,
551 commit_id: str | None,
552 snapshot_id: str | None,
553 pattern: str,
554 json_out: bool,
555 files_only: bool,
556 count_mode: bool,
557 context_lines: int,
558 ) -> None:
559 """Render search results to stdout in text or JSON format."""
560 if json_out:
561 payload: _ContentGrepJson = {
562 "source": source,
563 "commit_id": commit_id,
564 "snapshot_id": snapshot_id,
565 "pattern": pattern,
566 "total_files_matched": len(file_results),
567 "total_matches": total_matches,
568 "results": file_results,
569 }
570 print(json.dumps(payload, indent=2))
571 else:
572 for fr in file_results:
573 safe_path = sanitize_display(fr["path"])
574 if files_only:
575 print(safe_path)
576 elif count_mode:
577 print(f"{safe_path}:{fr['match_count']}")
578 else:
579 for m in fr["matches"]:
580 if context_lines > 0:
581 for ctx in m["context_before"]:
582 print(f"{safe_path}:{m['line_number']}-{sanitize_display(ctx)}")
583 print(f"{safe_path}:{m['line_number']}:{sanitize_display(m['text'])}")
584 for ctx in m["context_after"]:
585 print(f"{safe_path}:{m['line_number']}+{sanitize_display(ctx)}")
586 else:
587 print(f"{safe_path}:{m['line_number']}:{sanitize_display(m['text'])}")
File History 1 commit
sha256:cf6265cea8c21d9228d90dec13ef6ec2dab5103d466db9cc4590681832de4bf8 docs(KD-STAGING): sync governance after KD-6b DONE Human 13 days ago