gabriel / muse public
_midi_query.py python
574 lines 18.7 KB
Raw
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6 chore(tests): add docstring to tests/__init__.py so rc14 tr… Human 26 days ago
1 """Music-domain query DSL for the Muse VCS.
2
3 Allows agents and humans to query the commit history for musical content::
4
5 muse music-query "note.pitch_class == 'Eb' and bar == 12"
6 muse music-query "harmony.quality == 'dim' and bar == 8"
7 muse music-query "author == 'agent-x' and track == 'piano.mid'"
8 muse music-query "note.velocity > 80 and not bar == 4"
9 muse music-query "agent_id == 'counterpoint-bot'"
10
11 Grammar (EBNF)
12 --------------
13
14 query = or_expr
15 or_expr = and_expr ( 'or' and_expr )*
16 and_expr = not_expr ( 'and' not_expr )*
17 not_expr = 'not' not_expr | atom
18 atom = '(' query ')' | comparison
19 comparison = FIELD OP VALUE
20 FIELD = 'note.pitch' | 'note.pitch_class' | 'note.velocity'
21 | 'note.channel' | 'note.duration'
22 | 'bar' | 'track' | 'harmony.chord' | 'harmony.quality'
23 | 'author' | 'agent_id' | 'model_id' | 'toolchain_id'
24 OP = '==' | '!=' | '>' | '<' | '>=' | '<='
25 VALUE = QUOTED_STRING | NUMBER
26
27 Supported field paths
28 ---------------------
29
30 +---------------------+---------------------------------------------+
31 | Field | Resolves to |
32 +=====================+=============================================+
33 | note.pitch | any note's MIDI pitch (integer 0–127) |
34 +---------------------+---------------------------------------------+
35 | note.pitch_class | pitch class name ("C", "C#", …, "B") |
36 +---------------------+---------------------------------------------+
37 | note.velocity | MIDI velocity (0–127) |
38 +---------------------+---------------------------------------------+
39 | note.channel | MIDI channel (0–15) |
40 +---------------------+---------------------------------------------+
41 | note.duration | note duration in beats (float) |
42 +---------------------+---------------------------------------------+
43 | bar | 1-indexed bar number (assumes 4/4) |
44 +---------------------+---------------------------------------------+
45 | track | workspace-relative MIDI file path |
46 +---------------------+---------------------------------------------+
47 | harmony.chord | detected chord name ("Cmaj", "Fdim7", …) |
48 +---------------------+---------------------------------------------+
49 | harmony.quality | chord quality suffix ("maj", "min", "dim"…) |
50 +---------------------+---------------------------------------------+
51 | author | commit author string |
52 +---------------------+---------------------------------------------+
53 | agent_id | agent_id from commit provenance |
54 +---------------------+---------------------------------------------+
55 | model_id | model_id from commit provenance |
56 +---------------------+---------------------------------------------+
57 | toolchain_id | toolchain_id from commit provenance |
58 +---------------------+---------------------------------------------+
59 """
60
61 from __future__ import annotations
62
63 import logging
64 import pathlib
65 import re
66 from dataclasses import dataclass, field
67 from typing import Literal, TypedDict
68
69 from muse.core.object_store import read_object
70 from muse.core.store import CommitRecord, get_commit_snapshot_manifest, read_commit
71 from muse.plugins.midi._query import (
72 NoteInfo,
73 detect_chord,
74 notes_by_bar,
75 walk_commits_for_track,
76 )
77 from muse.plugins.midi.midi_diff import extract_notes
78
79
80 type _OpMap = dict[str, "_OpLiteral"]
81 logger = logging.getLogger(__name__)
82
83
84 # ---------------------------------------------------------------------------
85 # AST node dataclasses
86 # ---------------------------------------------------------------------------
87
88
89 @dataclass
90 class EqNode:
91 """Leaf comparison: ``field OP value``."""
92
93 field: str
94 op: Literal["==", "!=", ">", "<", ">=", "<="]
95 value: str | int | float
96
97
98 @dataclass
99 class AndNode:
100 """Logical AND of two sub-expressions."""
101
102 left: QueryNode
103 right: QueryNode
104
105
106 @dataclass
107 class OrNode:
108 """Logical OR of two sub-expressions."""
109
110 left: QueryNode
111 right: QueryNode
112
113
114 @dataclass
115 class NotNode:
116 """Logical NOT of a sub-expression."""
117
118 inner: QueryNode
119
120
121 QueryNode = EqNode | AndNode | OrNode | NotNode
122
123
124 # ---------------------------------------------------------------------------
125 # Query context and result types
126 # ---------------------------------------------------------------------------
127
128
129 @dataclass
130 class QueryContext:
131 """Data available to the evaluator for one bar in one track at one commit."""
132
133 commit: CommitRecord
134 track: str
135 bar: int
136 notes: list[NoteInfo]
137 chord: str
138 ticks_per_beat: int
139
140
141 class NoteDict(TypedDict):
142 """Serialisable representation of a note for query results."""
143
144 pitch: int
145 pitch_class: str
146 velocity: int
147 channel: int
148 beat: float
149 duration_beats: float
150
151
152 class QueryMatch(TypedDict):
153 """A single match returned by :func:`run_query`."""
154
155 commit_id: str
156 commit_short: str
157 commit_message: str
158 author: str
159 agent_id: str
160 committed_at: str
161 track: str
162 bar: int
163 notes: list[NoteDict]
164 chord: str
165 matched_on: str
166
167
168 # ---------------------------------------------------------------------------
169 # Tokenizer
170 # ---------------------------------------------------------------------------
171
172 _TOKEN_RE = re.compile(
173 r"""
174 (?P<LPAREN> \( ) |
175 (?P<RPAREN> \) ) |
176 (?P<OP> ==|!=|>=|<=|>|< ) |
177 (?P<KW> (?:and|or|not)(?![A-Za-z0-9_.]) ) |
178 (?P<STR> "(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*' ) |
179 (?P<NUM> -?\d+(?:\.\d+)? ) |
180 (?P<NAME> [A-Za-z_][A-Za-z0-9_.]* ) |
181 (?P<WS> \s+ )
182 """,
183 re.VERBOSE,
184 )
185
186 _OpLiteral = Literal["==", "!=", ">", "<", ">=", "<="]
187 _VALID_OPS: frozenset[str] = frozenset({"==", "!=", ">", "<", ">=", "<="})
188
189
190 @dataclass
191 class Token:
192 """A single lexed token."""
193
194 kind: str
195 value: str
196
197
198 def _tokenize(query: str) -> list[Token]:
199 """Convert *query* string to a flat list of :class:`Token` objects.
200
201 Whitespace tokens are discarded. Raises ``ValueError`` on unrecognised input.
202 """
203 tokens: list[Token] = []
204 for m in _TOKEN_RE.finditer(query):
205 kind = m.lastgroup
206 if kind is None or kind == "WS":
207 continue
208 tokens.append(Token(kind=kind, value=m.group()))
209 # Verify full coverage.
210 covered = sum(len(m.group()) for m in _TOKEN_RE.finditer(query) if m.lastgroup != "WS")
211 no_ws = re.sub(r"\s+", "", query)
212 if covered != len(no_ws):
213 raise ValueError(f"Unrecognised characters in query: {query!r}")
214 return tokens
215
216
217 # ---------------------------------------------------------------------------
218 # Recursive descent parser
219 # ---------------------------------------------------------------------------
220
221
222 class _Parser:
223 """Recursive descent parser for the music query DSL."""
224
225 def __init__(self, tokens: list[Token]) -> None:
226 self._tokens = tokens
227 self._pos = 0
228
229 def _peek(self) -> Token | None:
230 if self._pos < len(self._tokens):
231 return self._tokens[self._pos]
232 return None
233
234 def _consume(self, kind: str | None = None, value: str | None = None) -> Token:
235 tok = self._peek()
236 if tok is None:
237 raise ValueError("Unexpected end of query")
238 if kind is not None and tok.kind != kind:
239 raise ValueError(f"Expected {kind!r}, got {tok.kind!r} ({tok.value!r})")
240 if value is not None and tok.value != value:
241 raise ValueError(f"Expected {value!r}, got {tok.value!r}")
242 self._pos += 1
243 return tok
244
245 def parse(self) -> QueryNode:
246 node = self._or_expr()
247 if self._peek() is not None:
248 raise ValueError(f"Unexpected token: {self._peek()!r}")
249 return node
250
251 def _or_expr(self) -> QueryNode:
252 node = self._and_expr()
253 while (tok := self._peek()) is not None and tok.kind == "KW" and tok.value == "or":
254 self._consume()
255 right = self._and_expr()
256 node = OrNode(left=node, right=right)
257 return node
258
259 def _and_expr(self) -> QueryNode:
260 node = self._not_expr()
261 while (tok := self._peek()) is not None and tok.kind == "KW" and tok.value == "and":
262 self._consume()
263 right = self._not_expr()
264 node = AndNode(left=node, right=right)
265 return node
266
267 def _not_expr(self) -> QueryNode:
268 tok = self._peek()
269 if tok is not None and tok.kind == "KW" and tok.value == "not":
270 self._consume()
271 return NotNode(inner=self._not_expr())
272 return self._atom()
273
274 def _atom(self) -> QueryNode:
275 tok = self._peek()
276 if tok is None:
277 raise ValueError("Unexpected end of query in atom")
278 if tok.kind == "LPAREN":
279 self._consume("LPAREN")
280 node = self._or_expr()
281 self._consume("RPAREN")
282 return node
283 return self._comparison()
284
285 def _comparison(self) -> QueryNode:
286 field_tok = self._consume("NAME")
287 op_tok = self._consume("OP")
288 if op_tok.value not in _VALID_OPS:
289 raise ValueError(f"Invalid operator: {op_tok.value!r}")
290
291 val_tok = self._consume()
292 if val_tok.kind == "STR":
293 raw = val_tok.value[1:-1]
294 raw = raw.replace('\\"', '"').replace("\\'", "'")
295 value: str | int | float = raw
296 elif val_tok.kind == "NUM":
297 value = float(val_tok.value) if "." in val_tok.value else int(val_tok.value)
298 elif val_tok.kind == "NAME":
299 value = val_tok.value
300 else:
301 raise ValueError(f"Expected value, got {val_tok.kind!r}")
302
303 _op_map: _OpMap = {
304 "==": "==", "!=": "!=", ">": ">", "<": "<", ">=": ">=", "<=": "<=",
305 }
306 op_val = _op_map.get(op_tok.value)
307 if op_val is None:
308 raise ValueError(f"Invalid operator: {op_tok.value!r}")
309 return EqNode(
310 field=field_tok.value,
311 op=op_val,
312 value=value,
313 )
314
315
316 def parse_query(query_str: str) -> QueryNode:
317 """Parse a query string into an AST.
318
319 Args:
320 query_str: Music query expression.
321
322 Returns:
323 Root :data:`QueryNode` of the AST.
324
325 Raises:
326 ValueError: On parse error.
327 """
328 tokens = _tokenize(query_str)
329 return _Parser(tokens).parse()
330
331
332 # ---------------------------------------------------------------------------
333 # Evaluator
334 # ---------------------------------------------------------------------------
335
336
337 def _compare(actual: str | int | float, op: str, expected: str | int | float) -> bool:
338 """Apply a comparison operator to two values.
339
340 String comparisons use ``==`` / ``!=`` only (other operators raise).
341 Numeric comparisons support all six operators.
342 """
343 if isinstance(actual, str):
344 if op == "==":
345 return actual.lower() == str(expected).lower()
346 if op == "!=":
347 return actual.lower() != str(expected).lower()
348 raise ValueError(f"Operator {op!r} not supported for string values")
349
350 exp_num: float
351 if isinstance(expected, str):
352 try:
353 exp_num = float(expected)
354 except ValueError:
355 raise ValueError(f"Cannot compare numeric field to {expected!r}")
356 else:
357 exp_num = float(expected)
358
359 act_num = float(actual)
360 if op == "==":
361 return act_num == exp_num
362 if op == "!=":
363 return act_num != exp_num
364 if op == ">":
365 return act_num > exp_num
366 if op == "<":
367 return act_num < exp_num
368 if op == ">=":
369 return act_num >= exp_num
370 if op == "<=":
371 return act_num <= exp_num
372 raise ValueError(f"Unknown operator {op!r}")
373
374
375 def _resolve_field(field: str, ctx: QueryContext) -> list[str | int | float]:
376 """Resolve a field path to a list of candidate values from *ctx*.
377
378 Note fields return one value per note in the bar; all other fields
379 return a single-element list. The evaluator matches if *any* candidate
380 satisfies the predicate.
381 """
382 # --- Note fields ---
383 if field == "note.pitch":
384 return [n.pitch for n in ctx.notes]
385 if field == "note.pitch_class":
386 return [n.pitch_class_name for n in ctx.notes]
387 if field == "note.velocity":
388 return [n.velocity for n in ctx.notes]
389 if field == "note.channel":
390 return [n.channel for n in ctx.notes]
391 if field == "note.duration":
392 return [n.beat_duration for n in ctx.notes]
393 # --- Bar / track ---
394 if field == "bar":
395 return [ctx.bar]
396 if field == "track":
397 return [ctx.track]
398 # --- Harmony ---
399 if field == "harmony.chord":
400 return [ctx.chord]
401 if field == "harmony.quality":
402 for suffix in ("dim7", "maj7", "min7", "dom7", "sus2", "sus4", "aug", "dim", "maj", "min", "5"):
403 if ctx.chord.endswith(suffix):
404 return [suffix]
405 return [""]
406 # --- Commit provenance ---
407 if field == "author":
408 return [ctx.commit.author]
409 if field == "agent_id":
410 return [ctx.commit.agent_id]
411 if field == "model_id":
412 return [ctx.commit.model_id]
413 if field == "toolchain_id":
414 return [ctx.commit.toolchain_id]
415
416 raise ValueError(f"Unknown field: {field!r}")
417
418
419 def evaluate_node(node: QueryNode, ctx: QueryContext) -> bool:
420 """Recursively evaluate a query AST node against *ctx*.
421
422 Args:
423 node: The root (or sub) AST node.
424 ctx: Query context for the bar/track/commit being tested.
425
426 Returns:
427 ``True`` when the predicate matches, ``False`` otherwise.
428 """
429 if isinstance(node, EqNode):
430 try:
431 candidates = _resolve_field(node.field, ctx)
432 except ValueError:
433 return False
434 return any(_compare(c, node.op, node.value) for c in candidates)
435
436 if isinstance(node, AndNode):
437 return evaluate_node(node.left, ctx) and evaluate_node(node.right, ctx)
438
439 if isinstance(node, OrNode):
440 return evaluate_node(node.left, ctx) or evaluate_node(node.right, ctx)
441
442 if isinstance(node, NotNode):
443 return not evaluate_node(node.inner, ctx)
444
445 raise TypeError(f"Unknown AST node type: {type(node)}")
446
447
448 # ---------------------------------------------------------------------------
449 # Query runner
450 # ---------------------------------------------------------------------------
451
452
453 def run_query(
454 query_str: str,
455 root: pathlib.Path,
456 start_commit_id: str,
457 *,
458 track_filter: str | None = None,
459 from_commit_id: str | None = None,
460 max_commits: int = 10_000,
461 max_results: int = 1_000,
462 ) -> list[QueryMatch]:
463 """Evaluate a music query DSL expression over the commit history.
464
465 Walks the parent chain from *start_commit_id*, loading each MIDI track,
466 grouping notes by bar, and evaluating the query predicate against each
467 (commit, track, bar) triple.
468
469 Args:
470 query_str: Music query expression string.
471 root: Repository root.
472 start_commit_id: Start of the walk (inclusive; usually HEAD).
473 track_filter: Restrict search to a single MIDI file path.
474 from_commit_id: Stop before this commit ID (exclusive).
475 max_commits: Safety cap on commits walked.
476 max_results: Safety cap on results returned.
477
478 Returns:
479 List of :class:`QueryMatch` dicts, chronologically ordered
480 (oldest first).
481
482 Raises:
483 ValueError: When the query string cannot be parsed.
484 """
485 ast = parse_query(query_str)
486 results: list[QueryMatch] = []
487
488 commit_id: str | None = start_commit_id
489 seen: set[str] = set()
490 commits_walked = 0
491
492 while commit_id and commit_id not in seen and commits_walked < max_commits:
493 seen.add(commit_id)
494 commits_walked += 1
495
496 if commit_id == from_commit_id:
497 break
498
499 commit = read_commit(root, commit_id)
500 if commit is None:
501 break
502
503 manifest = get_commit_snapshot_manifest(root, commit_id) or {}
504
505 midi_paths = [
506 p for p in manifest
507 if p.lower().endswith(".mid")
508 and (track_filter is None or p == track_filter)
509 ]
510
511 for track_path in sorted(midi_paths):
512 obj_hash = manifest.get(track_path)
513 if obj_hash is None:
514 continue
515 raw = read_object(root, obj_hash)
516 if raw is None:
517 continue
518 try:
519 keys, tpb = extract_notes(raw)
520 except ValueError:
521 continue
522
523 notes = [NoteInfo.from_note_key(k, tpb) for k in keys]
524 bar_map = notes_by_bar(notes)
525
526 for bar_num, bar_notes in sorted(bar_map.items()):
527 pcs = frozenset(n.pitch_class for n in bar_notes)
528 chord = detect_chord(pcs)
529 ctx = QueryContext(
530 commit=commit,
531 track=track_path,
532 bar=bar_num,
533 notes=bar_notes,
534 chord=chord,
535 ticks_per_beat=tpb,
536 )
537 if evaluate_node(ast, ctx):
538 results.append(
539 QueryMatch(
540 commit_id=commit.commit_id,
541 commit_short=commit.commit_id[:8],
542 commit_message=commit.message,
543 author=commit.author,
544 agent_id=commit.agent_id,
545 committed_at=commit.committed_at.isoformat(),
546 track=track_path,
547 bar=bar_num,
548 notes=[
549 NoteDict(
550 pitch=n.pitch,
551 pitch_class=n.pitch_class_name,
552 velocity=n.velocity,
553 channel=n.channel,
554 beat=round(n.beat, 4),
555 duration_beats=round(n.beat_duration, 4),
556 )
557 for n in bar_notes
558 ],
559 chord=chord,
560 matched_on=query_str,
561 )
562 )
563 if len(results) >= max_results:
564 logger.warning(
565 "⚠️ music-query hit max_results=%d — truncating",
566 max_results,
567 )
568 results.reverse()
569 return results
570
571 commit_id = commit.parent_commit_id
572
573 results.reverse() # oldest-first
574 return results
File History 5 commits
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6 chore(tests): add docstring to tests/__init__.py so rc14 tr… Human 26 days ago
sha256:d8316ffae901be06347e16ab55be11868eb519dd16ade3e8aa16a99e662f7e62 baseline: rc14 re-baseline after rc3 store corruption recovery Human patch 26 days ago
sha256:50d413a635f57761c3fce1f70251b957b6423821525bf330747ef4007339222e Merge branch 'dev' into main Human 29 days ago
sha256:fb67fed5a4d3e40de84bdd163de94ef1386570bef1dd1a020a732c8a038962ce Merge branch 'dev' into main Human 48 days ago
sha256:1c4b3e3a9a1f300774c3ee662b572a698d5fd405bf765a71e6011a2e9c3eaaaa feat: Muse — version control for the agent era Human 100 days ago