symbol_diff.py python
613 lines 23.9 KB
Raw
sha256:cf6265cea8c21d9228d90dec13ef6ec2dab5103d466db9cc4590681832de4bf8 docs(KD-STAGING): sync governance after KD-6b DONE Human 13 days ago
1 """Symbol-level diff engine for the code domain plugin.
2
3 Produces typed :class:`~muse.domain.DomainOp` entries at symbol granularity
4 rather than file granularity. This allows Muse to report which *functions*
5 were added, removed, renamed, or modified — not just which *files* changed.
6
7 Operation types produced
8 ------------------------
9 ``InsertOp``
10 A symbol was added to the file (new function, class, etc.).
11
12 ``DeleteOp``
13 A symbol was removed from the file.
14
15 ``ReplaceOp``
16 A symbol's content changed. The ``old_summary`` / ``new_summary`` fields
17 describe the nature of the change:
18
19 - ``"renamed to <name>"`` — same body, different name (rename detected
20 via matching ``body_hash``).
21 - ``"signature changed"`` — same body, different signature.
22 - ``"implementation changed"`` — same signature, different body.
23 - ``"modified"`` — both signature and body changed.
24
25 ``MoveOp``
26 Reserved for intra-file positional moves (used when a symbol's address
27 is unchanged but it is detected elsewhere via ``content_id``).
28
29 Cross-file move detection
30 -------------------------
31 When a symbol disappears from one file and appears in another with an
32 identical ``content_id``, the diff engine annotates the ``DeleteOp`` and
33 ``InsertOp`` ``content_summary`` fields to indicate the move direction. No
34 special op type is introduced — the existing :class:`~muse.domain.InsertOp`
35 and :class:`~muse.domain.DeleteOp` suffice because their addresses already
36 encode the file path.
37
38 Algorithm
39 ---------
40 1. Partition symbol addresses into ``added``, ``removed``, and ``common``.
41 2. Build ``body_hash → address`` reverse maps for added and removed sets.
42 3. For each ``removed`` symbol:
43
44 a. If ``content_id`` matches an ``added`` symbol → **exact move/copy**
45 (same name, same body, different file).
46 b. Else if ``body_hash`` matches an ``added`` symbol → **rename**
47 (same body, different name).
48
49 4. Emit ``ReplaceOp`` for renames; pair the cross-file move partners via
50 ``content_summary``.
51 5. Emit ``DeleteOp`` for genuinely removed symbols.
52 6. Emit ``InsertOp`` for genuinely added symbols.
53 7. Emit ``ReplaceOp`` for symbols whose ``content_id`` changed.
54 """
55
56 from __future__ import annotations
57
58 import logging
59
60 from muse.core.store import Manifest
61 from muse.domain import DeleteOp, DomainOp, InsertOp, PatchOp, ReplaceOp
62 from muse.plugins.code.ast_parser import SymbolTree
63
64 logger = logging.getLogger(__name__)
65
66 type ContentIdMap = dict[str, str] # content_id or body_hash → address
67 type SetHashMap = dict[str, set[str]] # path → set of body hashes
68 type FileMoveMap = dict[str, str] # old_path → new_path
69 type ContentTupleMap = dict[str, tuple[str, int, list[DomainOp]]] # content_id → (addr, idx, ops)
70 type SymbolTreeMap = dict[str, SymbolTree] # file_path → symbol tree
71
72 _CHILD_DOMAIN = "code_symbols"
73
74
75 # ---------------------------------------------------------------------------
76 # Symbol-level diff within a single file
77 # ---------------------------------------------------------------------------
78
79
80 def diff_symbol_trees(
81 base: SymbolTree,
82 target: SymbolTree,
83 ) -> list[DomainOp]:
84 """Compute symbol-level ops transforming *base* into *target*.
85
86 Both trees must be scoped to the same file (their addresses share the
87 same ``"<file_path>::"`` prefix).
88
89 Args:
90 base: Symbol tree of the base (older) version of the file.
91 target: Symbol tree of the target (newer) version of the file.
92
93 Returns:
94 Ordered list of :class:`~muse.domain.DomainOp` entries.
95 """
96 base_addrs = set(base)
97 target_addrs = set(target)
98 added: set[str] = target_addrs - base_addrs
99 removed: set[str] = base_addrs - target_addrs
100 common: set[str] = base_addrs & target_addrs
101
102 # Reverse maps for rename / move detection.
103 added_by_content: ContentIdMap = {
104 target[a]["content_id"]: a for a in added
105 }
106 added_by_body: ContentIdMap = {
107 target[a]["body_hash"]: a for a in added
108 }
109
110 ops: list[DomainOp] = []
111 # Addresses claimed by rename / move detection — excluded from plain ops.
112 matched_removed: set[str] = set()
113 matched_added: set[str] = set()
114
115 # ── Pass 1: renames (same body_hash, different name) ──────────────────
116 for rem_addr in sorted(removed):
117 base_rec = base[rem_addr]
118
119 if base_rec["content_id"] in added_by_content:
120 # Exact content match at a different address → same symbol moved
121 # (intra-file positional moves don't produce a different address;
122 # this catches cross-file moves surfaced within a single-file diff
123 # when the caller slices the tree incorrectly — uncommon).
124 tgt_addr = added_by_content[base_rec["content_id"]]
125 tgt_rec = target[tgt_addr]
126 ops.append(ReplaceOp(
127 op="replace",
128 address=rem_addr,
129 position=None,
130 old_content_id=base_rec["content_id"],
131 new_content_id=tgt_rec["content_id"],
132 old_summary=f"{base_rec['kind']} {base_rec['name']}",
133 new_summary=f"moved to {tgt_rec['qualified_name']}",
134 ))
135 matched_removed.add(rem_addr)
136 matched_added.add(tgt_addr)
137
138 elif (
139 base_rec["body_hash"] in added_by_body
140 and added_by_body[base_rec["body_hash"]] not in matched_added
141 ):
142 # Same body, different name → rename.
143 tgt_addr = added_by_body[base_rec["body_hash"]]
144 tgt_rec = target[tgt_addr]
145 ops.append(ReplaceOp(
146 op="replace",
147 address=rem_addr,
148 position=None,
149 old_content_id=base_rec["content_id"],
150 new_content_id=tgt_rec["content_id"],
151 old_summary=f"{base_rec['kind']} {base_rec['name']}",
152 new_summary=f"renamed to {tgt_rec['name']}",
153 ))
154 matched_removed.add(rem_addr)
155 matched_added.add(tgt_addr)
156
157 # ── Pass 2: plain deletions ────────────────────────────────────────────
158 for rem_addr in sorted(removed - matched_removed):
159 rec = base[rem_addr]
160 ops.append(DeleteOp(
161 op="delete",
162 address=rem_addr,
163 position=None,
164 content_id=rec["content_id"],
165 content_summary=(
166 f"removed {rec['kind']} {rec['name']}"
167 f" L{rec['lineno']}–{rec['end_lineno']}"
168 ),
169 ))
170
171 # ── Pass 3: plain additions ────────────────────────────────────────────
172 for add_addr in sorted(added - matched_added):
173 rec = target[add_addr]
174 ops.append(InsertOp(
175 op="insert",
176 address=add_addr,
177 position=None,
178 content_id=rec["content_id"],
179 content_summary=(
180 f"added {rec['kind']} {rec['name']}"
181 f" L{rec['lineno']}–{rec['end_lineno']}"
182 ),
183 ))
184
185 # ── Pass 4: modifications ──────────────────────────────────────────────
186 for addr in sorted(common):
187 base_rec = base[addr]
188 tgt_rec = target[addr]
189 if base_rec["content_id"] == tgt_rec["content_id"]:
190 continue # unchanged
191
192 loc = f" L{tgt_rec['lineno']}–{tgt_rec['end_lineno']}"
193 if base_rec["body_hash"] == tgt_rec["body_hash"]:
194 old_summary = f"{base_rec['kind']} {base_rec['name']} (signature changed)"
195 new_summary = f"{tgt_rec['kind']} {tgt_rec['name']} (signature updated){loc}"
196 elif base_rec["signature_id"] == tgt_rec["signature_id"]:
197 old_summary = f"{base_rec['kind']} {base_rec['name']} (implementation)"
198 new_summary = f"{tgt_rec['kind']} {tgt_rec['name']} (implementation changed){loc}"
199 else:
200 old_summary = f"{base_rec['kind']} {base_rec['name']}"
201 new_summary = f"{tgt_rec['kind']} {tgt_rec['name']} (modified){loc}"
202
203 ops.append(ReplaceOp(
204 op="replace",
205 address=addr,
206 position=None,
207 old_content_id=base_rec["content_id"],
208 new_content_id=tgt_rec["content_id"],
209 old_summary=old_summary,
210 new_summary=new_summary,
211 ))
212
213 return ops
214
215
216 # ---------------------------------------------------------------------------
217 # Cross-file diff: build the full op list for a snapshot pair
218 # ---------------------------------------------------------------------------
219
220
221 def build_diff_ops(
222 base_files: Manifest,
223 target_files: Manifest,
224 base_trees: SymbolTreeMap,
225 target_trees: SymbolTreeMap,
226 ) -> list[DomainOp]:
227 """Build the complete op list transforming *base* snapshot into *target*.
228
229 For each changed file:
230
231 - **No symbol trees available**: coarse ``InsertOp`` / ``DeleteOp`` /
232 ``ReplaceOp`` at file level.
233 - **Symbol trees available for both sides**: ``PatchOp`` with symbol-level
234 ``child_ops``. If all symbols are unchanged (formatting-only change)
235 a ``ReplaceOp`` with ``"reformatted"`` summary is emitted instead.
236 - **Symbol tree available for one side only** (new or deleted file):
237 ``PatchOp`` listing each symbol individually.
238
239 Cross-file move annotation
240 --------------------------
241 After building per-file ops, a second pass checks whether any symbol
242 ``content_id`` appears in both a ``DeleteOp`` child op and an ``InsertOp``
243 child op across *different* files. When found, both ops' ``content_summary``
244 fields are annotated with the move direction.
245
246 Args:
247 base_files: ``{path: raw_bytes_hash}`` from the base snapshot.
248 target_files: ``{path: raw_bytes_hash}`` from the target snapshot.
249 base_trees: Symbol trees for changed base files, keyed by path.
250 target_trees: Symbol trees for changed target files, keyed by path.
251
252 Returns:
253 Ordered list of ``DomainOp`` entries.
254 """
255 base_paths = set(base_files)
256 target_paths = set(target_files)
257 added_paths = sorted(target_paths - base_paths)
258 removed_paths = sorted(base_paths - target_paths)
259 modified_paths = sorted(
260 p for p in base_paths & target_paths
261 if base_files[p] != target_files[p]
262 )
263
264 # Detect file-level move+edits before emitting per-file ops so we can
265 # suppress the plain added/removed ops for those paths.
266 move_map = _detect_file_move_edits(
267 added_paths, removed_paths, base_trees, target_trees
268 )
269 moved_old = set(move_map)
270 moved_new = set(move_map.values())
271
272 ops: list[DomainOp] = []
273
274 # ── Added files (excluding move+edit targets) ──────────────────────────
275 for path in added_paths:
276 if path in moved_new:
277 continue
278 tree = target_trees.get(path, {})
279 if tree:
280 child_ops: list[DomainOp] = [
281 InsertOp(
282 op="insert",
283 address=addr,
284 position=None,
285 content_id=rec["content_id"],
286 content_summary=(
287 f"added {rec['kind']} {rec['name']}"
288 f" L{rec['lineno']}–{rec['end_lineno']}"
289 ),
290 )
291 for addr, rec in sorted(tree.items())
292 ]
293 ops.append(_patch(path, child_ops))
294 else:
295 ops.append(InsertOp(
296 op="insert",
297 address=path,
298 position=None,
299 content_id=target_files[path],
300 content_summary=f"added {path}",
301 ))
302
303 # ── Removed files (excluding move+edit sources) ────────────────────────
304 for path in removed_paths:
305 if path in moved_old:
306 continue
307 tree = base_trees.get(path, {})
308 if tree:
309 child_ops = [
310 DeleteOp(
311 op="delete",
312 address=addr,
313 position=None,
314 content_id=rec["content_id"],
315 content_summary=(
316 f"removed {rec['kind']} {rec['name']}"
317 f" L{rec['lineno']}–{rec['end_lineno']}"
318 ),
319 )
320 for addr, rec in sorted(tree.items())
321 ]
322 ops.append(_patch(path, child_ops))
323 else:
324 ops.append(DeleteOp(
325 op="delete",
326 address=path,
327 position=None,
328 content_id=base_files[path],
329 content_summary=f"removed {path}",
330 ))
331
332 # ── Modified files ─────────────────────────────────────────────────────
333 for path in modified_paths:
334 base_tree = base_trees.get(path, {})
335 target_tree = target_trees.get(path, {})
336
337 if base_tree or target_tree:
338 child_ops = diff_symbol_trees(base_tree, target_tree)
339 if child_ops:
340 ops.append(_patch(path, child_ops))
341 else:
342 # All symbols have the same content_id — formatting-only change.
343 ops.append(ReplaceOp(
344 op="replace",
345 address=path,
346 position=None,
347 old_content_id=base_files[path],
348 new_content_id=target_files[path],
349 old_summary=f"{path} (before)",
350 new_summary=f"{path} (reformatted — no semantic change)",
351 ))
352 else:
353 ops.append(ReplaceOp(
354 op="replace",
355 address=path,
356 position=None,
357 old_content_id=base_files[path],
358 new_content_id=target_files[path],
359 old_summary=f"{path} (before)",
360 new_summary=f"{path} (after)",
361 ))
362
363 # ── Move+edit files ────────────────────────────────────────────────────
364 for old_path, new_path in sorted(move_map.items()):
365 old_tree = base_trees.get(old_path, {})
366 new_tree = target_trees.get(new_path, {})
367 # Strip the file-path prefix (everything up to and including "::") so
368 # that "math_utils.py::add" and "core_math.py::add" both normalise to
369 # "add" and land in the *common* bucket of diff_symbol_trees. Without
370 # this, every symbol looks deleted-and-added, producing spurious
371 # "moved to <name>" entries for functions that are actually unchanged.
372 old_tree_norm: SymbolTree = {
373 addr.split("::", 1)[-1]: rec for addr, rec in old_tree.items()
374 }
375 new_tree_norm: SymbolTree = {
376 addr.split("::", 1)[-1]: rec for addr, rec in new_tree.items()
377 }
378 child_ops = diff_symbol_trees(old_tree_norm, new_tree_norm)
379
380 n_added = sum(1 for o in child_ops if o["op"] == "insert")
381 n_removed = sum(1 for o in child_ops if o["op"] == "delete")
382 n_modified = sum(1 for o in child_ops if o["op"] == "replace")
383 sym_parts: list[str] = []
384 if n_added:
385 sym_parts.append(f"{n_added} added")
386 if n_removed:
387 sym_parts.append(f"{n_removed} removed")
388 if n_modified:
389 sym_parts.append(f"{n_modified} modified")
390 child_summary = f"moved from {old_path}"
391 if sym_parts:
392 child_summary += f"; {', '.join(sym_parts)}"
393
394 ops.append(PatchOp(
395 op="patch",
396 address=new_path,
397 from_address=old_path,
398 child_ops=child_ops,
399 child_domain=_CHILD_DOMAIN,
400 child_summary=child_summary,
401 ))
402
403 _annotate_cross_file_moves(ops)
404 return ops
405
406
407 def _patch(path: str, child_ops: list[DomainOp]) -> PatchOp:
408 """Wrap symbol child_ops in a file-level PatchOp."""
409 n_added = sum(1 for o in child_ops if o["op"] == "insert")
410 n_removed = sum(1 for o in child_ops if o["op"] == "delete")
411 n_modified = sum(1 for o in child_ops if o["op"] == "replace")
412 parts: list[str] = []
413 if n_added:
414 parts.append(f"{n_added} symbol{'s' if n_added > 1 else ''} added")
415 if n_removed:
416 parts.append(f"{n_removed} symbol{'s' if n_removed > 1 else ''} removed")
417 if n_modified:
418 parts.append(f"{n_modified} symbol{'s' if n_modified > 1 else ''} modified")
419 summary = ", ".join(parts) if parts else "no symbol changes"
420 return PatchOp(
421 op="patch",
422 address=path,
423 child_ops=child_ops,
424 child_domain=_CHILD_DOMAIN,
425 child_summary=summary,
426 )
427
428
429 def _detect_file_move_edits(
430 added_paths: list[str],
431 removed_paths: list[str],
432 base_trees: SymbolTreeMap,
433 target_trees: SymbolTreeMap,
434 min_overlap: float = 0.5,
435 ) -> FileMoveMap:
436 """Return a mapping of old_path → new_path for file-level move+edits.
437
438 A file is considered moved-and-edited when the two symbol trees share at
439 least ``min_overlap`` fraction of symbols by ``body_hash`` (computed
440 against the smaller set). This mirrors the symbol-level rename heuristic
441 applied cross-file.
442
443 Each old_path and new_path is used at most once (greedy, highest-overlap
444 pair wins when multiple candidates exist).
445
446 Args:
447 added_paths: Paths present in target but not in base.
448 removed_paths: Paths present in base but not in target.
449 base_trees: Symbol trees for changed base files.
450 target_trees: Symbol trees for changed target files.
451 min_overlap: Minimum fraction of matching body_hashes required.
452
453 Returns:
454 ``{old_path: new_path}`` for each detected move+edit pair.
455 """
456 base_hashes: SetHashMap = {
457 p: {rec["body_hash"] for rec in base_trees[p].values()}
458 for p in removed_paths
459 if p in base_trees and base_trees[p]
460 }
461 target_hashes: SetHashMap = {
462 p: {rec["body_hash"] for rec in target_trees[p].values()}
463 for p in added_paths
464 if p in target_trees and target_trees[p]
465 }
466
467 # Score all candidate pairs, then greedily assign best matches.
468 candidates: list[tuple[float, str, str]] = []
469 for old_path, old_h in base_hashes.items():
470 for new_path, new_h in target_hashes.items():
471 common = old_h & new_h
472 if not common:
473 continue
474 overlap = len(common) / min(len(old_h), len(new_h))
475 if overlap >= min_overlap:
476 candidates.append((overlap, old_path, new_path))
477
478 candidates.sort(key=lambda t: t[0], reverse=True)
479
480 moves: FileMoveMap = {}
481 used_old: set[str] = set()
482 used_new: set[str] = set()
483 for _, old_path, new_path in candidates:
484 if old_path in used_old or new_path in used_new:
485 continue
486 moves[old_path] = new_path
487 used_old.add(old_path)
488 used_new.add(new_path)
489
490 return moves
491
492
493 def _annotate_cross_file_moves(ops: list[DomainOp]) -> None:
494 """Annotate DeleteOp/InsertOp pairs that represent cross-file symbol moves.
495
496 Mutates the ``content_summary`` of matching ops in place. A move is
497 detected when:
498
499 - A ``DeleteOp`` child op (inside a ``PatchOp``) has the same
500 ``content_id`` as an ``InsertOp`` child op in a *different* file's
501 ``PatchOp``.
502
503 This is a best-effort annotation pass — it does not change the semantic
504 meaning of the ops, only their human-readable summaries.
505 """
506 # Collect child op references: content_id → (file_path, op_index_in_patch)
507 # We need mutable access so work with lists rather than immutable tuples.
508 delete_by_content: ContentTupleMap = {}
509 insert_by_content: ContentTupleMap = {}
510
511 for op in ops:
512 if op["op"] != "patch":
513 continue
514 file_path = op["address"]
515 for i, child in enumerate(op["child_ops"]):
516 if child["op"] == "delete":
517 delete_by_content[child["content_id"]] = (file_path, i, op["child_ops"])
518 elif child["op"] == "insert":
519 insert_by_content[child["content_id"]] = (file_path, i, op["child_ops"])
520
521 for content_id, (del_file, del_idx, del_children) in delete_by_content.items():
522 if content_id not in insert_by_content:
523 continue
524 ins_file, ins_idx, ins_children = insert_by_content[content_id]
525 if del_file == ins_file:
526 continue # Same file — not a cross-file move.
527
528 del_op = del_children[del_idx]
529 ins_op = ins_children[ins_idx]
530 # Narrow to the expected op kinds before accessing kind-specific fields.
531 if del_op["op"] != "delete" or ins_op["op"] != "insert":
532 continue
533 # Annotate both sides with move direction.
534 del_children[del_idx] = DeleteOp(
535 op="delete",
536 address=del_op["address"],
537 position=del_op["position"],
538 content_id=del_op["content_id"],
539 content_summary=f"{del_op['content_summary']} → moved to {ins_file}",
540 )
541 ins_children[ins_idx] = InsertOp(
542 op="insert",
543 address=ins_op["address"],
544 position=ins_op["position"],
545 content_id=ins_op["content_id"],
546 content_summary=f"{ins_op['content_summary']} ← moved from {del_file}",
547 )
548
549
550 # ---------------------------------------------------------------------------
551 # Summary helpers
552 # ---------------------------------------------------------------------------
553
554
555 def delta_summary(ops: list[DomainOp]) -> str:
556 """Produce a human-readable one-line summary of a list of ops.
557
558 Counts file-level and symbol-level operations separately.
559
560 Args:
561 ops: Top-level op list from :func:`build_diff_ops`.
562
563 Returns:
564 A concise summary string (e.g. ``"2 files modified (5 symbols)"``)
565 or ``"no changes"`` for an empty list.
566 """
567 files_added = sum(1 for o in ops if o["op"] == "insert" and "::" not in o["address"])
568 files_removed = sum(1 for o in ops if o["op"] == "delete" and "::" not in o["address"])
569 files_modified = sum(1 for o in ops if o["op"] in ("replace", "patch") and "::" not in o["address"])
570 dirs_renamed = sum(1 for o in ops if o["op"] == "directory_rename")
571
572 # Count child-level symbol ops.
573 symbols_added = 0
574 symbols_removed = 0
575 symbols_modified = 0
576 for op in ops:
577 if op["op"] == "patch":
578 for child in op["child_ops"]:
579 if child["op"] == "insert":
580 symbols_added += 1
581 elif child["op"] == "delete":
582 symbols_removed += 1
583 elif child["op"] in ("replace", "move"):
584 symbols_modified += 1
585 elif op["op"] == "replace" and "::" not in op["address"]:
586 # File-level replace with no symbol breakdown.
587 pass
588
589 parts: list[str] = []
590 file_parts: list[str] = []
591 if files_added:
592 file_parts.append(f"{files_added} added")
593 if files_removed:
594 file_parts.append(f"{files_removed} removed")
595 if files_modified:
596 file_parts.append(f"{files_modified} modified")
597 if file_parts:
598 parts.append(f"{', '.join(file_parts)} file{'s' if sum([files_added, files_removed, files_modified]) != 1 else ''}")
599
600 sym_parts: list[str] = []
601 if symbols_added:
602 sym_parts.append(f"{symbols_added} added")
603 if symbols_removed:
604 sym_parts.append(f"{symbols_removed} removed")
605 if symbols_modified:
606 sym_parts.append(f"{symbols_modified} modified")
607 if sym_parts:
608 parts.append(f"{', '.join(sym_parts)} symbol{'s' if sum([symbols_added, symbols_removed, symbols_modified]) != 1 else ''}")
609
610 if dirs_renamed:
611 parts.append(f"{dirs_renamed} director{'ies' if dirs_renamed != 1 else 'y'} renamed")
612
613 return ", ".join(parts) if parts else "no changes"
File History 1 commit
sha256:cf6265cea8c21d9228d90dec13ef6ec2dab5103d466db9cc4590681832de4bf8 docs(KD-STAGING): sync governance after KD-6b DONE Human 13 days ago