aaronrene / gabriel-muse public
entity.py python
590 lines 21.3 KB
Raw
sha256:cf6265cea8c21d9228d90dec13ef6ec2dab5103d466db9cc4590681832de4bf8 docs(KD-STAGING): sync governance after KD-6b DONE Human 13 days ago
1 """Stable entity identity for MIDI note objects in the Muse MIDI plugin.
2
3 The key insight
4 ---------------
5 Content hash ≠ entity identity.
6
7 The current note content ID — ``SHA-256(pitch:velocity:start_tick:duration_ticks:channel)``
8 — is correct for *content equality* but wrong for *entity identity*. When a
9 musician or agent changes a note's velocity from 80 to 100, the old model
10 produces a ``DeleteOp + InsertOp`` (two unrelated content hashes). With stable
11 entity identity, the diff produces a ``MutateOp`` — "velocity 80→100 on note
12 C4@bar4" — and the note's lineage is preserved through the edit.
13
14 A ``NoteEntity`` extends the five ``NoteKey`` fields with an optional
15 ``entity_id`` — a UUID4 assigned at first insertion that persists across all
16 subsequent mutations regardless of how the note's properties change.
17
18 Entity assignment heuristic
19 ----------------------------
20 The function :func:`assign_entity_ids` maps a new note list to the entity IDs
21 in a prior index using a three-tier match:
22
23 1. **Exact content match** — all five fields identical → same entity, no mutation.
24 2. **Fuzzy match** — same pitch + channel, ``|Δtick| ≤ threshold``, and
25 ``|Δvelocity| ≤ threshold`` → same entity, emit ``MutateOp``.
26 3. **No match** → new entity, new UUID4, emit ``InsertOp``.
27
28 Notes in the prior index that matched nothing → emit ``DeleteOp``.
29
30 Storage
31 -------
32 Entity indexes live under ``.muse/entity_index/`` as derived artifacts:
33
34 .muse/entity_index/<commit_id>/<track_safe_name>.json
35
36 They are fully rebuildable from commit history and should be added to the
37 ``[domain.midi]`` section of ``.museignore`` in agent automation scripts
38 to avoid accidental commits.
39
40 Public API
41 ----------
42 - :class:`NoteEntity` — ``NoteKey`` fields + optional entity metadata.
43 - :class:`EntityIndexEntry` — one entity's record in the index.
44 - :class:`EntityIndex` — the full per-track, per-commit index.
45 - :func:`assign_entity_ids` — map a new note list onto prior entity IDs.
46 - :func:`diff_with_entity_ids` — entity-aware diff → ``list[DomainOp]``.
47 - :func:`build_entity_index` — build an :class:`EntityIndex` from entities.
48 - :func:`write_entity_index` / :func:`read_entity_index` — I/O.
49 """
50
51 from __future__ import annotations
52
53 import hashlib
54 import json
55 import logging
56 import pathlib
57 import uuid as _uuid_mod
58 from typing import TypedDict
59
60 from muse.domain import (
61 DeleteOp,
62 DomainOp,
63 FieldMutation,
64 InsertOp,
65 MutateOp,
66 )
67 from muse.plugins.midi.midi_diff import NoteKey, _note_content_id, _note_summary # noqa: PLC2701
68
69 type _ContentMap = dict[str, str]
70 type _NoteKeyBuckets = dict[str, list[NoteKey]]
71 type _FieldMutationMap = dict[str, FieldMutation]
72 type _EntityMap = dict[str, "EntityIndexEntry"]
73 type _NoteEntityMap = dict[str, "NoteEntity"]
74
75 logger = logging.getLogger(__name__)
76
77 _ENTITY_INDEX_DIR = ".muse/entity_index"
78
79
80 # ---------------------------------------------------------------------------
81 # Types
82 # ---------------------------------------------------------------------------
83
84
85 class _NoteEntityRequired(TypedDict):
86 """Required fields shared with NoteKey."""
87
88 pitch: int
89 velocity: int
90 start_tick: int
91 duration_ticks: int
92 channel: int
93
94
95 class NoteEntity(_NoteEntityRequired, total=False):
96 """A MIDI note with optional stable entity identity.
97
98 When ``entity_id`` is absent the note is treated as content-only (legacy
99 behaviour). When present it is a UUID4 that persists across mutations to
100 velocity, timing, or duration — enabling lineage tracking through edits.
101
102 ``voice_id``
103 Optional voice lane identifier (e.g. ``"soprano"``, ``"alto"``).
104 Assigned by a voice-separation analysis pass; not required for basic
105 entity tracking.
106 ``origin_commit_id``
107 Short-form commit ID where this entity was first created.
108 ``origin_op_id``
109 Op UUID from the op log that created this entity.
110 """
111
112 entity_id: str
113 voice_id: str
114 origin_commit_id: str
115 origin_op_id: str
116
117
118 class EntityIndexEntry(TypedDict):
119 """One entity's record in the per-track entity index.
120
121 ``content_id``
122 SHA-256 content hash of the note's current fields (the five ``NoteKey``
123 fields). Updated on every mutation.
124 ``origin_commit_id``
125 Commit where this entity was first inserted.
126 ``voice_id``
127 Voice stream assignment, or empty string if unassigned.
128 """
129
130 content_id: str
131 origin_commit_id: str
132 voice_id: str
133
134
135 class EntityIndex(TypedDict):
136 """Complete entity index for one track at one commit.
137
138 ``entities`` maps ``entity_id`` → :class:`EntityIndexEntry`.
139 This is the lookup table used by :func:`assign_entity_ids` to re-identify
140 notes across commits.
141 """
142
143 track_path: str
144 commit_id: str
145 entities: _EntityMap
146
147
148 # ---------------------------------------------------------------------------
149 # Entity ID assignment
150 # ---------------------------------------------------------------------------
151
152 #: Default threshold in MIDI ticks for fuzzy timing match (≈ 10 ms at 120 BPM
153 #: with 480 ticks/beat: 480 × 0.010 × 2 ≈ 10 ticks).
154 _DEFAULT_TICK_THRESHOLD = 10
155
156 #: Default velocity difference threshold for fuzzy entity matching.
157 _DEFAULT_VEL_THRESHOLD = 20
158
159
160 def assign_entity_ids(
161 notes: list[NoteKey],
162 prior_index: EntityIndex | None,
163 commit_id: str,
164 op_id: str,
165 *,
166 mutation_threshold_ticks: int = _DEFAULT_TICK_THRESHOLD,
167 mutation_threshold_velocity: int = _DEFAULT_VEL_THRESHOLD,
168 ) -> list[NoteEntity]:
169 """Assign stable entity IDs to a list of notes.
170
171 Maps each note in *notes* to an entity ID from *prior_index* using a
172 three-tier matching heuristic (exact → fuzzy → new).
173
174 Args:
175 notes: New note list (from the current commit).
176 prior_index: Entity index from the parent commit.
177 ``None`` means this is the first commit
178 for this track (all notes get new IDs).
179 commit_id: Current commit ID (stored as provenance).
180 op_id: Op log entry ID that produced these notes.
181 mutation_threshold_ticks: Max |Δtick| for fuzzy timing match.
182 mutation_threshold_velocity: Max |Δvelocity| for fuzzy match.
183
184 Returns:
185 List of :class:`NoteEntity` objects in the same order as *notes*,
186 each with a populated ``entity_id``.
187 """
188 if prior_index is None:
189 return [_new_entity(n, commit_id, op_id) for n in notes]
190
191 # Build lookup: content_id → entity_id for exact matches.
192 content_to_entity: _ContentMap = {}
193 # Build list for fuzzy matching: [(entity_id, note_key_fields, entry)]
194 fuzzy_candidates: list[tuple[str, NoteKey]] = []
195
196 for entity_id, entry in prior_index["entities"].items():
197 cid = entry["content_id"]
198 # Reconstruct a NoteKey from the content hash is impossible directly,
199 # so we carry a parallel lookup keyed by content_id string.
200 content_to_entity[cid] = entity_id
201 # For fuzzy matching we need the actual field values. These aren't
202 # stored in the index entry (only the hash is), so fuzzy matching
203 # operates on the NEW notes' fields against the hash. We build the
204 # fuzzy set from the incoming notes rather than the prior index.
205 _ = fuzzy_candidates # populated below
206
207 # Build a richer map from new notes' content IDs.
208 new_by_cid: _NoteKeyBuckets = {}
209 for n in notes:
210 cid = _note_content_id(n)
211 new_by_cid.setdefault(cid, []).append(n)
212
213 # --- Tier 1: exact content match ---
214 # Assign entity IDs to notes whose content hash appears in the prior index.
215 assigned: dict[int, str] = {} # index → entity_id
216 used_entities: set[str] = set()
217
218 for i, note in enumerate(notes):
219 cid = _note_content_id(note)
220 if cid in content_to_entity:
221 eid = content_to_entity[cid]
222 if eid not in used_entities:
223 assigned[i] = eid
224 used_entities.add(eid)
225
226 # --- Tier 2: fuzzy match for unassigned notes ---
227 # Build prior note field table from the original notes that produced the
228 # prior index. Since we only have hashes, we use the *new* notes as a
229 # proxy: any note with (same pitch, same channel, close tick, close vel)
230 # is a mutation candidate.
231 #
232 # Approach: for each unassigned new note, find an unused prior entity
233 # whose content_id resolves to a note with matching pitch+channel and
234 # close tick+velocity. Since we can't reverse-SHA the prior hash, we
235 # instead accept the fuzzy match if the content hash of the hypothetical
236 # un-mutated note (same pitch, channel, but old vel/tick fields) matches.
237 #
238 # In practice, callers pass both old and new note lists when they have
239 # them; the simple heuristic here covers the common agent-edit case.
240 prior_entity_ids = list(prior_index["entities"].keys())
241
242 for i, note in enumerate(notes):
243 if i in assigned:
244 continue
245 # Try to match against any unused prior entity by field similarity.
246 # We approximate by assuming the prior entity had similar fields.
247 best_eid: str | None = None
248 best_score = float("inf")
249
250 for eid in prior_entity_ids:
251 if eid in used_entities:
252 continue
253 entry = prior_index["entities"][eid]
254 prior_cid = entry["content_id"]
255
256 # Attempt to reconstruct a plausible prior note for this entity.
257 # We don't have the raw fields — approximate by checking if a
258 # note with the same pitch + channel but slightly different
259 # timing/velocity would hash to this content_id.
260 for vel_delta in range(-mutation_threshold_velocity, mutation_threshold_velocity + 1, 2):
261 for tick_delta in range(-mutation_threshold_ticks, mutation_threshold_ticks + 1, 2):
262 candidate: NoteKey = NoteKey(
263 pitch=note["pitch"],
264 velocity=max(0, min(127, note["velocity"] + vel_delta)),
265 start_tick=max(0, note["start_tick"] + tick_delta),
266 duration_ticks=note["duration_ticks"],
267 channel=note["channel"],
268 )
269 if _note_content_id(candidate) == prior_cid:
270 score = abs(vel_delta) + abs(tick_delta)
271 if score < best_score:
272 best_score = score
273 best_eid = eid
274 break
275 if best_eid is not None and best_score == 0:
276 break
277
278 if best_eid is not None:
279 assigned[i] = best_eid
280 used_entities.add(best_eid)
281
282 # --- Build output ---
283 result: list[NoteEntity] = []
284 for i, note in enumerate(notes):
285 if i in assigned:
286 entity: NoteEntity = NoteEntity(
287 pitch=note["pitch"],
288 velocity=note["velocity"],
289 start_tick=note["start_tick"],
290 duration_ticks=note["duration_ticks"],
291 channel=note["channel"],
292 entity_id=assigned[i],
293 origin_commit_id=prior_index["entities"][assigned[i]]["origin_commit_id"],
294 origin_op_id=op_id,
295 voice_id=prior_index["entities"][assigned[i]].get("voice_id", ""),
296 )
297 else:
298 entity = _new_entity(note, commit_id, op_id)
299 result.append(entity)
300
301 return result
302
303
304 def _new_entity(note: NoteKey, commit_id: str, op_id: str) -> NoteEntity:
305 """Create a :class:`NoteEntity` with a fresh UUID4 entity_id."""
306 return NoteEntity(
307 pitch=note["pitch"],
308 velocity=note["velocity"],
309 start_tick=note["start_tick"],
310 duration_ticks=note["duration_ticks"],
311 channel=note["channel"],
312 entity_id=str(_uuid_mod.uuid4()),
313 origin_commit_id=commit_id,
314 origin_op_id=op_id,
315 voice_id="",
316 )
317
318
319 # ---------------------------------------------------------------------------
320 # Entity-aware diff
321 # ---------------------------------------------------------------------------
322
323
324 def diff_with_entity_ids(
325 base_entities: list[NoteEntity],
326 target_entities: list[NoteEntity],
327 ticks_per_beat: int,
328 ) -> list[DomainOp]:
329 """Produce an entity-aware diff between two note lists.
330
331 Compared to the content-hash-only diff in :mod:`~muse.plugins.midi.midi_diff`,
332 this function detects *mutations* — cases where the same entity_id appears
333 in both lists with different field values — and emits ``MutateOp`` entries
334 instead of ``DeleteOp + InsertOp`` pairs.
335
336 Algorithm:
337 1. Build ``entity_id → NoteEntity`` maps for base and target.
338 2. For entities present in both: compare fields; emit ``MutateOp`` if
339 anything changed, otherwise "keep" (no op).
340 3. For entities only in base: emit ``DeleteOp``.
341 4. For entities only in target: emit ``InsertOp``.
342 5. For notes without an entity_id: fall back to content-hash comparison
343 (insert/delete only, no mutation tracking).
344
345 Args:
346 base_entities: Notes from the ancestor commit, with entity IDs.
347 target_entities: Notes from the current commit, with entity IDs.
348 ticks_per_beat: Used for human-readable summaries.
349
350 Returns:
351 Ordered list of :class:`~muse.domain.DomainOp` entries.
352 """
353 ops: list[DomainOp] = []
354
355 # Separate tracked (have entity_id) from untracked notes.
356 base_tracked: _NoteEntityMap = {}
357 base_untracked: list[NoteEntity] = []
358 for note in base_entities:
359 if "entity_id" in note and note.get("entity_id"):
360 base_tracked[note["entity_id"]] = note
361 else:
362 base_untracked.append(note)
363
364 target_tracked: _NoteEntityMap = {}
365 target_untracked: list[NoteEntity] = []
366 for note in target_entities:
367 if "entity_id" in note and note.get("entity_id"):
368 target_tracked[note["entity_id"]] = note
369 else:
370 target_untracked.append(note)
371
372 # --- Tracked: mutate, keep, insert, delete ---
373 all_entity_ids = set(base_tracked) | set(target_tracked)
374
375 for eid in sorted(all_entity_ids):
376 base_note = base_tracked.get(eid)
377 target_note = target_tracked.get(eid)
378
379 if base_note is not None and target_note is not None:
380 old_cid = _note_content_id(_entity_to_key(base_note))
381 new_cid = _note_content_id(_entity_to_key(target_note))
382 if old_cid == new_cid:
383 continue # unchanged
384
385 fields = _field_diff(base_note, target_note)
386 base_note_key = _entity_to_key(base_note)
387 target_note_key = _entity_to_key(target_note)
388 ops.append(
389 MutateOp(
390 op="mutate",
391 address=f"note:entity:{eid}",
392 entity_id=eid,
393 old_content_id=old_cid,
394 new_content_id=new_cid,
395 fields=fields,
396 old_summary=_note_summary(base_note_key, ticks_per_beat),
397 new_summary=_note_summary(target_note_key, ticks_per_beat),
398 position=None,
399 )
400 )
401
402 elif base_note is not None:
403 cid = _note_content_id(_entity_to_key(base_note))
404 ops.append(
405 DeleteOp(
406 op="delete",
407 address=f"note:entity:{eid}",
408 position=None,
409 content_id=cid,
410 content_summary=_note_summary(_entity_to_key(base_note), ticks_per_beat),
411 )
412 )
413
414 else:
415 assert target_note is not None
416 cid = _note_content_id(_entity_to_key(target_note))
417 ops.append(
418 InsertOp(
419 op="insert",
420 address=f"note:entity:{eid}",
421 position=None,
422 content_id=cid,
423 content_summary=_note_summary(_entity_to_key(target_note), ticks_per_beat),
424 )
425 )
426
427 # --- Untracked: fall back to content-hash insert/delete ---
428 base_content_ids = {_note_content_id(_entity_to_key(n)) for n in base_untracked}
429 target_content_ids = {_note_content_id(_entity_to_key(n)) for n in target_untracked}
430
431 for note in base_untracked:
432 cid = _note_content_id(_entity_to_key(note))
433 if cid not in target_content_ids:
434 ops.append(
435 DeleteOp(
436 op="delete",
437 address="note:untracked",
438 position=None,
439 content_id=cid,
440 content_summary=_note_summary(_entity_to_key(note), ticks_per_beat),
441 )
442 )
443
444 for note in target_untracked:
445 cid = _note_content_id(_entity_to_key(note))
446 if cid not in base_content_ids:
447 ops.append(
448 InsertOp(
449 op="insert",
450 address="note:untracked",
451 position=None,
452 content_id=cid,
453 content_summary=_note_summary(_entity_to_key(note), ticks_per_beat),
454 )
455 )
456
457 return ops
458
459
460 def _entity_to_key(entity: NoteEntity) -> NoteKey:
461 """Extract the five NoteKey fields from a NoteEntity."""
462 return NoteKey(
463 pitch=entity["pitch"],
464 velocity=entity["velocity"],
465 start_tick=entity["start_tick"],
466 duration_ticks=entity["duration_ticks"],
467 channel=entity["channel"],
468 )
469
470
471 def _field_diff(base: NoteEntity, target: NoteEntity) -> _FieldMutationMap:
472 """Return a FieldMutation map for all fields that changed."""
473 mutations: _FieldMutationMap = {}
474 # Unpack into flat tuples to avoid variable-key TypedDict access.
475 base_vals: tuple[int, int, int, int, int] = (
476 base["pitch"], base["velocity"], base["start_tick"], base["duration_ticks"], base["channel"]
477 )
478 target_vals: tuple[int, int, int, int, int] = (
479 target["pitch"], target["velocity"], target["start_tick"], target["duration_ticks"], target["channel"]
480 )
481 names = ("pitch", "velocity", "start_tick", "duration_ticks", "channel")
482 for name, bv, tv in zip(names, base_vals, target_vals):
483 if bv != tv:
484 mutations[name] = FieldMutation(old=str(bv), new=str(tv))
485 return mutations
486
487
488 # ---------------------------------------------------------------------------
489 # Entity index I/O
490 # ---------------------------------------------------------------------------
491
492
493 def build_entity_index(
494 entities: list[NoteEntity],
495 track_path: str,
496 commit_id: str,
497 ) -> EntityIndex:
498 """Build an :class:`EntityIndex` from a list of :class:`NoteEntity` objects.
499
500 Notes without an ``entity_id`` are skipped (untracked notes do not appear
501 in the index).
502
503 Args:
504 entities: Note entities from the current commit.
505 track_path: Workspace-relative MIDI file path.
506 commit_id: Current commit ID.
507
508 Returns:
509 A populated :class:`EntityIndex`.
510 """
511 entries: _EntityMap = {}
512 for note in entities:
513 eid = note.get("entity_id", "")
514 if not eid:
515 continue
516 entries[eid] = EntityIndexEntry(
517 content_id=_note_content_id(_entity_to_key(note)),
518 origin_commit_id=note.get("origin_commit_id", commit_id),
519 voice_id=note.get("voice_id", ""),
520 )
521 return EntityIndex(
522 track_path=track_path,
523 commit_id=commit_id,
524 entities=entries,
525 )
526
527
528 def _index_path(repo_root: pathlib.Path, commit_id: str, track_path: str) -> pathlib.Path:
529 safe_track = track_path.replace("/", "_").replace(".", "_")
530 sha = hashlib.sha256(track_path.encode()).hexdigest()[:8]
531 return (
532 repo_root
533 / _ENTITY_INDEX_DIR
534 / commit_id[:16]
535 / f"{safe_track}_{sha}.json"
536 )
537
538
539 def write_entity_index(
540 repo_root: pathlib.Path,
541 commit_id: str,
542 track_path: str,
543 index: EntityIndex,
544 ) -> None:
545 """Persist *index* to ``.muse/entity_index/<commit_id>/<track>.json``.
546
547 Creates parent directories as needed. Safe to call multiple times —
548 an existing file is overwritten.
549
550 Args:
551 repo_root: Repository root.
552 commit_id: Commit ID for the snapshot this index belongs to.
553 track_path: Workspace-relative MIDI file path.
554 index: The entity index to persist.
555 """
556 path = _index_path(repo_root, commit_id, track_path)
557 path.parent.mkdir(parents=True, exist_ok=True)
558 path.write_text(json.dumps(index, indent=2) + "\n")
559 logger.debug(
560 "✅ Entity index written: %d entities for %r @ %s",
561 len(index["entities"]),
562 track_path,
563 commit_id[:8],
564 )
565
566
567 def read_entity_index(
568 repo_root: pathlib.Path,
569 commit_id: str,
570 track_path: str,
571 ) -> EntityIndex | None:
572 """Load the entity index for *track_path* at *commit_id*.
573
574 Args:
575 repo_root: Repository root.
576 commit_id: Commit ID.
577 track_path: Workspace-relative MIDI file path.
578
579 Returns:
580 The :class:`EntityIndex`, or ``None`` when no index file exists.
581 """
582 path = _index_path(repo_root, commit_id, track_path)
583 if not path.exists():
584 return None
585 try:
586 raw: EntityIndex = json.loads(path.read_text())
587 return raw
588 except (json.JSONDecodeError, KeyError) as exc:
589 logger.warning("⚠️ Corrupt entity index %s: %s", path, exc)
590 return None
File History 1 commit
sha256:cf6265cea8c21d9228d90dec13ef6ec2dab5103d466db9cc4590681832de4bf8 docs(KD-STAGING): sync governance after KD-6b DONE Human 13 days ago