aaronrene / gabriel-muse public
_predicate.py python
411 lines 13.5 KB
Raw
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6 chore(tests): add docstring to tests/__init__.py so rc14 tr… Human 33 days ago
1 """Predicate DSL parser for ``muse query`` and ``muse query-history``.
2
3 Grammar (v2)
4 ============
5
6 .. code-block:: text
7
8 expr = or_expr
9 or_expr = and_expr ( "OR" and_expr )*
10 and_expr = not_expr ( and_expr )* # implicit AND / explicit AND
11 not_expr = "NOT" primary | primary
12 primary = "(" expr ")" | atom
13 atom = KEY OP VALUE
14
15 KEY = [a-zA-Z_][a-zA-Z_0-9]*
16 OP = "~=" | "^=" | "$=" | ">=" | "<=" | "!=" | "="
17 VALUE = double-quoted-string | bare-word
18
19 Supported keys
20 --------------
21
22 Snapshot-local (available with or without ``--all-commits``):
23
24 kind function | class | method | variable | import | …
25 language Python | Go | Rust | …
26 name bare symbol name
27 qualified_name dotted name (e.g. User.save)
28 file file path
29 hash content_id prefix (exact-body match)
30 body_hash body_hash prefix
31 signature_id signature_id prefix
32 lineno_gt symbol starts after line N (integer)
33 lineno_lt symbol starts before line N (integer)
34
35 Example queries
36 ---------------
37
38 kind=function language=Python name~=validate
39 (kind=function OR kind=method) name^=_
40 NOT kind=import file~=billing
41 kind=class name~=Service language=Python
42 lineno_gt=100 lineno_lt=200 file=src/billing.py
43
44 The parser is a hand-written recursive descent parser — no external
45 dependencies, no regex-based hacks. All parsing errors raise
46 ``PredicateError`` with a human-readable message including the position
47 in the input string.
48
49 New keys (v2.1)
50 ---------------
51
52 size_gt symbol body exceeds N lines (end_lineno - lineno > N)
53 size_lt symbol body shorter than N lines (end_lineno - lineno < N)
54
55 Example::
56
57 kind=function size_gt=50 # oversized functions (refactor targets)
58 kind=function size_lt=3 NOT name^=_ # suspiciously tiny public functions
59
60 Security note
61 -------------
62 Expression nesting is capped at ``_MAX_DEPTH`` (64) levels to prevent
63 stack overflow from crafted predicate strings.
64 """
65
66 from __future__ import annotations
67
68 import re
69 import logging
70 from collections.abc import Callable
71
72 from muse.plugins.code._query import language_of
73 from muse.plugins.code.ast_parser import SymbolRecord
74
75 logger = logging.getLogger(__name__)
76
77 # Maximum parenthesis nesting depth — prevents stack overflow from crafted input.
78 _MAX_DEPTH = 64
79
80 # Signature: (file_path: str, rec: SymbolRecord) -> bool
81 Predicate = Callable[[str, SymbolRecord], bool]
82
83
84 class PredicateError(ValueError):
85 """Raised when a predicate string cannot be parsed or evaluated."""
86
87 # ---------------------------------------------------------------------------
88 # Tokeniser
89 # ---------------------------------------------------------------------------
90
91 _TOKEN_SPEC = [
92 ("LPAREN", r"\("),
93 ("RPAREN", r"\)"),
94 ("OR", r"\bOR\b"),
95 ("NOT", r"\bNOT\b"),
96 ("AND", r"\bAND\b"),
97 ("ATOM", r'[a-zA-Z_][a-zA-Z_0-9]*(?:~=|\^=|\$=|>=|<=|!=|=)"[^"]*"'),
98 ("ATOM", r'[a-zA-Z_][a-zA-Z_0-9]*(?:~=|\^=|\$=|>=|<=|!=|=)[^\s()]+'),
99 ("WS", r"\s+"),
100 ]
101
102 _TOKEN_RE = re.compile("|".join(f"(?P<{name}_{i}>{pat})" for i, (name, pat) in enumerate(_TOKEN_SPEC)))
103
104
105 class _Token:
106 __slots__ = ("kind", "value", "pos")
107
108 def __init__(self, kind: str, value: str, pos: int) -> None:
109 self.kind = kind
110 self.value = value
111 self.pos = pos
112
113 def __repr__(self) -> str:
114 return f"Token({self.kind!r}, {self.value!r})"
115
116
117 def _tokenise(text: str) -> list[_Token]:
118 tokens: list[_Token] = []
119 pos = 0
120 while pos < len(text):
121 m = _TOKEN_RE.match(text, pos)
122 if m is None:
123 raise PredicateError(
124 f"Unexpected character at position {pos}: {text[pos]!r}"
125 )
126 kind_raw = m.lastgroup or ""
127 # Strip the numeric suffix we added.
128 kind = kind_raw.rsplit("_", 1)[0]
129 if kind != "WS":
130 tokens.append(_Token(kind, m.group(), pos))
131 pos = m.end()
132 return tokens
133
134
135 # ---------------------------------------------------------------------------
136 # Atom parser
137 # ---------------------------------------------------------------------------
138
139 _OP_RE = re.compile(r"(~=|\^=|\$=|>=|<=|!=|=)")
140 _VALID_KEYS = frozenset({
141 "kind", "language", "name", "qualified_name", "file",
142 "hash", "body_hash", "signature_id",
143 "lineno_gt", "lineno_lt",
144 "size_gt", "size_lt",
145 })
146
147
148 def _parse_int_value(key: str, value: str) -> int:
149 """Parse *value* as an integer, raising ``PredicateError`` on failure."""
150 try:
151 return int(value)
152 except ValueError:
153 raise PredicateError(f"{key} requires an integer value (got '{value}').")
154
155
156 def _parse_atom(atom_str: str) -> Predicate:
157 """Parse a single ``key OP value`` atom into a predicate callable."""
158 m = _OP_RE.search(atom_str)
159 if m is None:
160 raise PredicateError(
161 f"Cannot parse predicate '{atom_str}'. "
162 "Expected: key=value, key~=value, key^=value, key$=value, key!=value."
163 )
164 op = m.group(1)
165 key = atom_str[:m.start()].strip()
166 value = atom_str[m.end():]
167 # Strip surrounding double quotes.
168 if value.startswith('"') and value.endswith('"'):
169 value = value[1:-1]
170 if key not in _VALID_KEYS:
171 raise PredicateError(
172 f"Unknown predicate key '{key}'. "
173 f"Valid keys: {', '.join(sorted(_VALID_KEYS))}."
174 )
175
176 value_lower = value.lower()
177
178 # ── String-match keys ────────────────────────────────────────────────────
179
180 def _str_match(field: str) -> bool:
181 f = field.lower()
182 if op == "=":
183 return f == value_lower
184 if op == "~=":
185 return value_lower in f
186 if op == "^=":
187 return f.startswith(value_lower)
188 if op == "$=":
189 return f.endswith(value_lower)
190 if op == "!=":
191 return f != value_lower
192 return False
193
194 if key == "kind":
195 def _kind(file_path: str, rec: SymbolRecord) -> bool:
196 return _str_match(rec["kind"])
197 return _kind
198
199 if key == "language":
200 def _language(file_path: str, rec: SymbolRecord) -> bool:
201 return _str_match(language_of(file_path))
202 return _language
203
204 if key == "name":
205 def _name(file_path: str, rec: SymbolRecord) -> bool:
206 return _str_match(rec["name"])
207 return _name
208
209 if key == "qualified_name":
210 def _qname(file_path: str, rec: SymbolRecord) -> bool:
211 return _str_match(rec["qualified_name"])
212 return _qname
213
214 if key == "file":
215 def _file(file_path: str, rec: SymbolRecord) -> bool:
216 return _str_match(file_path)
217 return _file
218
219 if key == "hash":
220 prefix = value.lower()
221 def _hash(file_path: str, rec: SymbolRecord, _p: str = prefix) -> bool:
222 return rec["content_id"].startswith(_p)
223 return _hash
224
225 if key == "body_hash":
226 prefix = value.lower()
227 def _body_hash(file_path: str, rec: SymbolRecord, _p: str = prefix) -> bool:
228 return rec["body_hash"].startswith(_p)
229 return _body_hash
230
231 if key == "signature_id":
232 prefix = value.lower()
233 def _sig_id(file_path: str, rec: SymbolRecord, _p: str = prefix) -> bool:
234 return rec["signature_id"].startswith(_p)
235 return _sig_id
236
237 # ── Integer-range keys ───────────────────────────────────────────────────
238
239 if key == "lineno_gt":
240 threshold = _parse_int_value("lineno_gt", value)
241 def _lineno_gt(file_path: str, rec: SymbolRecord, _t: int = threshold) -> bool:
242 return rec["lineno"] > _t
243 return _lineno_gt
244
245 if key == "lineno_lt":
246 threshold = _parse_int_value("lineno_lt", value)
247 def _lineno_lt(file_path: str, rec: SymbolRecord, _t: int = threshold) -> bool:
248 return rec["lineno"] < _t
249 return _lineno_lt
250
251 if key == "size_gt":
252 threshold = _parse_int_value("size_gt", value)
253 def _size_gt(file_path: str, rec: SymbolRecord, _t: int = threshold) -> bool:
254 return (rec["end_lineno"] - rec["lineno"]) > _t
255 return _size_gt
256
257 if key == "size_lt":
258 threshold = _parse_int_value("size_lt", value)
259 def _size_lt(file_path: str, rec: SymbolRecord, _t: int = threshold) -> bool:
260 return (rec["end_lineno"] - rec["lineno"]) < _t
261 return _size_lt
262
263 # Should be unreachable — all valid keys handled above.
264 raise PredicateError(f"Internal error: unhandled key '{key}'.")
265
266
267 # ---------------------------------------------------------------------------
268 # Recursive descent parser
269 # ---------------------------------------------------------------------------
270
271
272 class _Parser:
273 """Recursive descent parser for the predicate grammar."""
274
275 def __init__(self, tokens: list[_Token]) -> None:
276 self._tokens = tokens
277 self._pos = 0
278 self._depth = 0 # parenthesis nesting depth
279
280 def _peek(self) -> _Token | None:
281 if self._pos < len(self._tokens):
282 return self._tokens[self._pos]
283 return None
284
285 def _consume(self, kind: str | None = None) -> _Token:
286 tok = self._peek()
287 if tok is None:
288 raise PredicateError("Unexpected end of predicate expression.")
289 if kind is not None and tok.kind != kind:
290 raise PredicateError(
291 f"Expected {kind!r} at position {tok.pos}, got {tok.kind!r} ({tok.value!r})."
292 )
293 self._pos += 1
294 return tok
295
296 def parse(self) -> Predicate:
297 pred = self._parse_or()
298 if self._peek() is not None:
299 tok = self._peek()
300 assert tok is not None
301 raise PredicateError(
302 f"Unexpected token at position {tok.pos}: {tok.value!r}"
303 )
304 return pred
305
306 def _parse_or(self) -> Predicate:
307 left = self._parse_and()
308 peek = self._peek()
309 while peek is not None and peek.kind == "OR":
310 self._consume("OR")
311 right = self._parse_and()
312 left_cap = left
313 right_cap = right
314 def _or(fp: str, rec: SymbolRecord, _l: Predicate = left_cap, _r: Predicate = right_cap) -> bool:
315 return _l(fp, rec) or _r(fp, rec)
316 left = _or
317 peek = self._peek()
318 return left
319
320 def _parse_and(self) -> Predicate:
321 left = self._parse_not()
322 while True:
323 tok = self._peek()
324 # Continue AND-chaining: next token is ATOM, LPAREN, or explicit AND.
325 if tok is None:
326 break
327 if tok.kind in ("RPAREN", "OR"):
328 break
329 if tok.kind == "AND":
330 self._consume("AND")
331 right = self._parse_not()
332 left_cap = left
333 right_cap = right
334 def _and(fp: str, rec: SymbolRecord, _l: Predicate = left_cap, _r: Predicate = right_cap) -> bool:
335 return _l(fp, rec) and _r(fp, rec)
336 left = _and
337 return left
338
339 def _parse_not(self) -> Predicate:
340 peek = self._peek()
341 if peek is not None and peek.kind == "NOT":
342 self._consume("NOT")
343 inner = self._parse_primary()
344 inner_cap = inner
345 def _not(fp: str, rec: SymbolRecord, _i: Predicate = inner_cap) -> bool:
346 return not _i(fp, rec)
347 return _not
348 return self._parse_primary()
349
350 def _parse_primary(self) -> Predicate:
351 tok = self._peek()
352 if tok is None:
353 raise PredicateError("Unexpected end of expression — expected predicate or '('.")
354 if tok.kind == "LPAREN":
355 self._depth += 1
356 if self._depth > _MAX_DEPTH:
357 raise PredicateError(
358 f"Predicate expression nested too deeply"
359 f" (max {_MAX_DEPTH} levels)."
360 )
361 self._consume("LPAREN")
362 inner = self._parse_or()
363 self._consume("RPAREN")
364 self._depth -= 1
365 return inner
366 if tok.kind == "ATOM":
367 self._consume("ATOM")
368 return _parse_atom(tok.value)
369 raise PredicateError(
370 f"Unexpected token at position {tok.pos}: {tok.value!r}. "
371 "Expected a predicate (key=value) or '('."
372 )
373
374
375 # ---------------------------------------------------------------------------
376 # Public API
377 # ---------------------------------------------------------------------------
378
379
380 def parse_query(tokens_or_str: str | list[str]) -> Predicate:
381 """Parse a predicate query expression into a single callable.
382
383 Args:
384 tokens_or_str: Either a single query string (which may contain OR/NOT/
385 parentheses) or a list of strings that are AND'd together
386 (legacy multi-argument style).
387
388 Returns:
389 A ``Predicate`` callable ``(file_path, SymbolRecord) -> bool``.
390
391 Raises:
392 PredicateError: If the query cannot be parsed.
393 """
394 if isinstance(tokens_or_str, list):
395 # Legacy: list of atoms → implicit AND of all.
396 if not tokens_or_str:
397 # Match everything.
398 return lambda _fp, _rec: True
399 combined = " ".join(tokens_or_str)
400 else:
401 combined = tokens_or_str
402
403 combined = combined.strip()
404 if not combined:
405 return lambda _fp, _rec: True
406
407 tokens = _tokenise(combined)
408 if not tokens:
409 return lambda _fp, _rec: True
410
411 return _Parser(tokens).parse()
File History 2 commits
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6 chore(tests): add docstring to tests/__init__.py so rc14 tr… Human 33 days ago
sha256:d8316ffae901be06347e16ab55be11868eb519dd16ade3e8aa16a99e662f7e62 baseline: rc14 re-baseline after rc3 store corruption recovery Human patch 33 days ago