gabriel / muse public
symbols.py python
550 lines 19.7 KB
Raw
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6 chore(tests): add docstring to tests/__init__.py so rc14 tr… Human 26 days ago
1 """Knowtation symbol model — note / field / section address extraction.
2
3 This module implements the three symbol kinds that Muse tracks for Knowtation
4 vault notes. Symbol addresses follow the same ``file::symbol`` convention used
5 by the code domain (``src/utils.py::calculate_total``) so that generic Muse
6 commands (``muse code symbols``, ``muse code impact``, ``muse code blame``) can
7 be made domain-aware in a later phase without changing their output schema.
8
9 Symbol kinds
10 ------------
11 ``note``
12 The note itself. Address = vault-relative path (e.g. ``inbox/foo.md``).
13 There is exactly one ``NoteSymbol`` per note.
14
15 ``field``
16 A frontmatter key-value pair. Address = ``{path}::field:{key}``
17 (e.g. ``inbox/foo.md::field:source``). One ``FieldSymbol`` per non-empty
18 field in the parsed frontmatter.
19
20 ``section``
21 A Markdown heading and the body text beneath it. Address =
22 ``{path}::section:{level}:{title}`` (e.g.
23 ``inbox/foo.md::section:2:Background``). Level is the number of ``#``
24 characters (1–6). A special ``level=0`` section (title ``"(preamble)"``)
25 is generated for body text that appears before the first heading.
26
27 Section splitter
28 ----------------
29 Deterministic Python port of the ``splitByHeadingOrSize`` heading regex from
30 ``knowtation/lib/chunk.mjs``. The original JS regex is::
31
32 /(?=^#{2,3}\\s)/m
33
34 This module generalises to all heading levels (H1–H6) for the symbol model
35 (``chunk.mjs`` only uses H2/H3 for chunking). The port does **not** call
36 Node.js — it uses :mod:`re` with ``re.MULTILINE``.
37
38 Address conventions
39 -------------------
40 - Separator between file and symbol: ``::``
41 - ``field`` prefix + colon: ``field:``
42 - ``section`` prefix + colon: ``section:``
43 - Level immediately follows ``section:``: ``section:2:Background``
44 - All components are left unescaped so addresses are human-readable in logs.
45 """
46
47 from __future__ import annotations
48
49 import re
50 from dataclasses import dataclass, field
51 from typing import Any, Literal
52
53 from muse.plugins.knowtation.parser import ParsedFrontmatter, parse_frontmatter
54
55 # ──────────────────────────────────────────────────────────────────────────────
56 # Address helpers
57 # ──────────────────────────────────────────────────────────────────────────────
58
59 #: Separator between vault-relative path and the symbol qualifier.
60 ADDRESS_SEP: str = "::"
61
62 #: Prefix for frontmatter field addresses (after the ``::`` separator).
63 FIELD_PREFIX: str = "field"
64
65 #: Prefix for section addresses (after the ``::`` separator).
66 SECTION_PREFIX: str = "section"
67
68
69 def make_note_address(path: str) -> str:
70 """Return the address for a note symbol (the vault-relative path itself).
71
72 Args:
73 path: Vault-relative file path (e.g. ``"inbox/foo.md"``).
74
75 Returns:
76 The same path string used as the note's canonical address.
77 """
78 return path
79
80
81 def make_field_address(path: str, key: str) -> str:
82 """Return the address for a frontmatter field symbol.
83
84 Args:
85 path: Vault-relative file path.
86 key: Frontmatter key name (e.g. ``"source"``).
87
88 Returns:
89 Address string, e.g. ``"inbox/foo.md::field:source"``.
90 """
91 return f"{path}{ADDRESS_SEP}{FIELD_PREFIX}:{key}"
92
93
94 def make_section_address(path: str, level: int, title: str) -> str:
95 """Return the address for a section symbol.
96
97 Args:
98 path: Vault-relative file path.
99 level: Heading level (0 = preamble, 1–6 = H1–H6).
100 title: Heading text without the ``#`` characters.
101
102 Returns:
103 Address string, e.g. ``"inbox/foo.md::section:2:Background"``.
104 """
105 return f"{path}{ADDRESS_SEP}{SECTION_PREFIX}:{level}:{title}"
106
107
108 def parse_address(address: str) -> dict[str, str]:
109 """Decompose a symbol address into its components.
110
111 Args:
112 address: A full symbol address string.
113
114 Returns:
115 Dict with keys ``"path"``, ``"kind"``, and (for field/section) their
116 qualifier keys. ``"kind"`` is ``"note"``, ``"field"``, or
117 ``"section"``.
118
119 Examples::
120
121 parse_address("inbox/foo.md")
122 → {"path": "inbox/foo.md", "kind": "note"}
123
124 parse_address("inbox/foo.md::field:source")
125 → {"path": "inbox/foo.md", "kind": "field", "key": "source"}
126
127 parse_address("inbox/foo.md::section:2:Background")
128 → {"path": "inbox/foo.md", "kind": "section",
129 "level": "2", "title": "Background"}
130 """
131 if ADDRESS_SEP not in address:
132 return {"path": address, "kind": "note"}
133
134 path, qualifier = address.split(ADDRESS_SEP, 1)
135
136 if qualifier.startswith(f"{FIELD_PREFIX}:"):
137 key = qualifier[len(FIELD_PREFIX) + 1:]
138 return {"path": path, "kind": "field", "key": key}
139
140 if qualifier.startswith(f"{SECTION_PREFIX}:"):
141 rest = qualifier[len(SECTION_PREFIX) + 1:]
142 level_str, _, title = rest.partition(":")
143 return {"path": path, "kind": "section", "level": level_str, "title": title}
144
145 return {"path": path, "kind": "unknown", "qualifier": qualifier}
146
147
148 # ──────────────────────────────────────────────────────────────────────────────
149 # Symbol dataclasses
150 # ──────────────────────────────────────────────────────────────────────────────
151
152
153 @dataclass
154 class NoteSymbol:
155 """Top-level symbol representing the vault note as a whole.
156
157 There is exactly one ``NoteSymbol`` per note, always the first entry
158 returned by :func:`extract_symbols`.
159 """
160
161 kind: Literal["note"] = field(default="note", init=False)
162 path: str = ""
163 address: str = ""
164 #: Display title — from frontmatter ``title`` field, first H1 heading, or filename.
165 title: str = ""
166 #: Project slug from frontmatter (``None`` when absent).
167 project: str | None = None
168 #: Tags from frontmatter (empty list when absent).
169 tags: list[str] = field(default_factory=list)
170
171 def to_dict(self) -> dict[str, Any]:
172 return {
173 "kind": self.kind,
174 "address": self.address,
175 "path": self.path,
176 "title": self.title,
177 "project": self.project,
178 "tags": self.tags,
179 }
180
181
182 @dataclass
183 class FieldSymbol:
184 """Symbol for a single frontmatter key-value pair.
185
186 One ``FieldSymbol`` is emitted for every non-``None`` / non-empty field
187 in the parsed frontmatter, including ``extra`` keys.
188 """
189
190 kind: Literal["field"] = field(default="field", init=False)
191 path: str = ""
192 key: str = ""
193 value: Any = None
194 #: Python type name of the value (``"str"``, ``"list"``, ``"bool"``, …).
195 value_type: str = ""
196 address: str = ""
197
198 def to_dict(self) -> dict[str, Any]:
199 return {
200 "kind": self.kind,
201 "address": self.address,
202 "path": self.path,
203 "key": self.key,
204 "value_type": self.value_type,
205 }
206
207
208 @dataclass
209 class SectionSymbol:
210 """Symbol for one Markdown heading section.
211
212 The body text of each section runs from (and including) the heading line
213 up to (but not including) the next heading of any level, or end-of-note.
214
215 A ``level=0`` section with ``title="(preamble)"`` is generated for body
216 text that appears before the first heading.
217 """
218
219 kind: Literal["section"] = field(default="section", init=False)
220 path: str = ""
221 #: Heading level: 0 = preamble, 1–6 = H1–H6.
222 level: int = 0
223 #: Heading text stripped of ``#`` characters and surrounding whitespace.
224 title: str = ""
225 #: 0-based index among all sections in this note (preamble = index 0 if present).
226 index: int = 0
227 #: 1-based line number of the heading within the note body (not counting frontmatter).
228 line: int = 1
229 #: Full section text (heading line + body until next heading).
230 body: str = ""
231 address: str = ""
232
233 def to_dict(self) -> dict[str, Any]:
234 return {
235 "kind": self.kind,
236 "address": self.address,
237 "path": self.path,
238 "level": self.level,
239 "title": self.title,
240 "index": self.index,
241 "line": self.line,
242 }
243
244
245 #: Union type for all symbol kinds.
246 Symbol = NoteSymbol | FieldSymbol | SectionSymbol
247
248
249 # ──────────────────────────────────────────────────────────────────────────────
250 # Section splitter — deterministic Python port of chunk.mjs heading regex
251 # ──────────────────────────────────────────────────────────────────────────────
252
253 # Original JS (chunk.mjs): /(?=^#{2,3}\s)/m — lookahead on ## or ###
254 # Extended to H1–H6 for the full symbol model.
255 # ATX heading: optional closing hashes stripped (CommonMark §4.2 closed ATX).
256 # (?:#+\s*)? at the end removes patterns like "## Title ##".
257 _HEADING_RE: re.Pattern[str] = re.compile(
258 r"^(#{1,6})\s+(.*?)\s*(?:#+\s*)?$", re.MULTILINE
259 )
260
261 #: Markdown extensions recognised as note files.
262 _NOTE_EXTENSIONS: frozenset[str] = frozenset({".md", ".markdown", ".mdx"})
263
264
265 def _strip_frontmatter(text: str) -> tuple[str, int]:
266 """Return (body_text, body_start_line) with frontmatter removed.
267
268 ``body_start_line`` is the 1-based line number in *text* where the body
269 begins (used to offset section line numbers so they refer to the raw file).
270
271 Args:
272 text: Full note text decoded from bytes.
273
274 Returns:
275 Tuple of (body_without_frontmatter, 1-based_line_offset).
276 """
277 if not text.startswith("---"):
278 return text, 1
279
280 rest = text[3:]
281 end = -1
282 for delim in ("\n---", "\n..."):
283 pos = rest.find(delim)
284 if pos != -1 and (end == -1 or pos < end):
285 end = pos
286
287 if end == -1:
288 return text, 1
289
290 # Skip past the closing delimiter line
291 close_end = end + len("\n---")
292 if close_end < len(rest) and rest[close_end] == "\n":
293 close_end += 1
294
295 body = rest[close_end:]
296 # Count lines consumed by the frontmatter block
297 fm_text = text[: 3 + close_end]
298 fm_lines = fm_text.count("\n")
299 return body, fm_lines + 1
300
301
302 def split_sections(body: str) -> list[tuple[int, str, str, int]]:
303 """Split a Markdown body into sections by heading.
304
305 Deterministic Python port of ``splitByHeadingOrSize`` from
306 ``knowtation/lib/chunk.mjs``, extended from H2/H3 to H1–H6 for symbol
307 granularity.
308
309 A ``(0, "(preamble)", text, 1)`` entry is prepended when text exists
310 before the first heading.
311
312 Args:
313 body: Markdown body text with frontmatter already stripped.
314
315 Returns:
316 List of ``(level, title, section_body, 1_based_line)`` tuples.
317 Empty list when *body* is blank.
318 """
319 if not body.strip():
320 return []
321
322 headings = list(_HEADING_RE.finditer(body))
323
324 results: list[tuple[int, str, str, int]] = []
325
326 # Preamble — text before the first heading
327 preamble_end = headings[0].start() if headings else len(body)
328 preamble = body[:preamble_end]
329 if preamble.strip():
330 results.append((0, "(preamble)", preamble, 1))
331
332 for i, match in enumerate(headings):
333 level = len(match.group(1))
334 title = match.group(2)
335 start = match.start()
336 end = headings[i + 1].start() if i + 1 < len(headings) else len(body)
337 section_body = body[start:end]
338 # Line number within the body (1-based)
339 body_line = body[:start].count("\n") + 1
340 results.append((level, title, section_body, body_line))
341
342 return results
343
344
345 # ──────────────────────────────────────────────────────────────────────────────
346 # Field extraction
347 # ──────────────────────────────────────────────────────────────────────────────
348
349 #: Frontmatter fields to emit as FieldSymbols. All known non-empty fields
350 #: from ParsedFrontmatter.to_dict() plus extra keys.
351 def _field_symbols_from_frontmatter(
352 path: str, fm: ParsedFrontmatter
353 ) -> list[FieldSymbol]:
354 """Return one :class:`FieldSymbol` per non-empty frontmatter field.
355
356 Args:
357 path: Vault-relative note path.
358 fm: Parsed frontmatter from :func:`~parser.parse_frontmatter`.
359
360 Returns:
361 List of :class:`FieldSymbol` instances in SPEC §2 field order.
362 """
363 symbols: list[FieldSymbol] = []
364 for key, value in fm.to_dict().items():
365 if value is None:
366 continue
367 if isinstance(value, list) and not value:
368 continue
369 if isinstance(value, bool) and not value:
370 continue
371 value_type = type(value).__name__
372 symbols.append(
373 FieldSymbol(
374 path=path,
375 key=key,
376 value=value,
377 value_type=value_type,
378 address=make_field_address(path, key),
379 )
380 )
381 return symbols
382
383
384 # ──────────────────────────────────────────────────────────────────────────────
385 # Note title resolution
386 # ──────────────────────────────────────────────────────────────────────────────
387
388
389 def _resolve_note_title(
390 path: str, fm: ParsedFrontmatter | None, body: str
391 ) -> str:
392 """Resolve the display title for a note.
393
394 Priority:
395 1. ``title`` field in frontmatter (SPEC §2.1).
396 2. First H1 heading in the body.
397 3. Filename stem (without extension).
398
399 Args:
400 path: Vault-relative note path.
401 fm: Parsed frontmatter (or ``None`` when absent).
402 body: Note body with frontmatter stripped.
403
404 Returns:
405 Display title string.
406 """
407 if fm is not None and fm.title:
408 return fm.title
409
410 # First H1 heading
411 h1 = _HEADING_RE.search(body)
412 if h1 and len(h1.group(1)) == 1:
413 return h1.group(2)
414
415 # Filename stem
416 stem = path.rsplit("/", 1)[-1]
417 for ext in _NOTE_EXTENSIONS:
418 if stem.endswith(ext):
419 stem = stem[: -len(ext)]
420 break
421 return stem
422
423
424 # ──────────────────────────────────────────────────────────────────────────────
425 # Main extraction API
426 # ──────────────────────────────────────────────────────────────────────────────
427
428
429 def extract_symbols(path: str, content: bytes) -> list[Symbol]:
430 """Extract all symbols from a vault note's raw bytes.
431
432 Returns symbols in this order:
433 1. One :class:`NoteSymbol` (always present).
434 2. One :class:`FieldSymbol` per non-empty frontmatter field.
435 3. One :class:`SectionSymbol` per heading section (including preamble when present).
436
437 Args:
438 path: Vault-relative file path (e.g. ``"inbox/foo.md"``).
439 content: Raw bytes of the note file.
440
441 Returns:
442 List of :class:`Symbol` objects. Never raises — returns only the
443 ``NoteSymbol`` when parsing or splitting fails.
444 """
445 symbols: list[Symbol] = []
446
447 # ── Parse ────────────────────────────────────────────────────────────────
448 try:
449 text = content.decode("utf-8", errors="replace")
450 except Exception:
451 text = ""
452
453 fm = parse_frontmatter(content)
454 body, _body_line_offset = _strip_frontmatter(text)
455
456 # ── NoteSymbol ───────────────────────────────────────────────────────────
457 note_title = _resolve_note_title(path, fm, body)
458 note_sym = NoteSymbol(
459 path=path,
460 address=make_note_address(path),
461 title=note_title,
462 project=fm.project if fm else None,
463 tags=list(fm.tags) if fm else [],
464 )
465 symbols.append(note_sym)
466
467 # ── FieldSymbols ─────────────────────────────────────────────────────────
468 if fm is not None:
469 symbols.extend(_field_symbols_from_frontmatter(path, fm))
470
471 # ── SectionSymbols ───────────────────────────────────────────────────────
472 try:
473 sections = split_sections(body)
474 except Exception:
475 sections = []
476
477 for idx, (level, title, sec_body, body_line) in enumerate(sections):
478 symbols.append(
479 SectionSymbol(
480 path=path,
481 level=level,
482 title=title,
483 index=idx,
484 line=body_line,
485 body=sec_body,
486 address=make_section_address(path, level, title),
487 )
488 )
489
490 return symbols
491
492
493 def extract_sections(path: str, content: bytes) -> list[SectionSymbol]:
494 """Return only the :class:`SectionSymbol` entries for a note.
495
496 Convenience wrapper around :func:`extract_symbols` for callers that only
497 need the section graph (e.g. diff and merge engines).
498
499 Args:
500 path: Vault-relative file path.
501 content: Raw bytes of the note file.
502
503 Returns:
504 List of :class:`SectionSymbol` instances in document order.
505 """
506 return [s for s in extract_symbols(path, content) if isinstance(s, SectionSymbol)]
507
508
509 def extract_fields(path: str, content: bytes) -> list[FieldSymbol]:
510 """Return only the :class:`FieldSymbol` entries for a note.
511
512 Args:
513 path: Vault-relative file path.
514 content: Raw bytes of the note file.
515
516 Returns:
517 List of :class:`FieldSymbol` instances in SPEC §2 field order.
518 """
519 return [s for s in extract_symbols(path, content) if isinstance(s, FieldSymbol)]
520
521
522 def symbols_to_json(symbols: list[Symbol]) -> dict[str, Any]:
523 """Serialise a symbol list to the ``muse code symbols --json`` output shape.
524
525 The schema matches the ``code`` domain's JSON output so the same tooling
526 can consume vault symbols without special-casing:
527
528 .. code-block:: json
529
530 {
531 "domain": "knowtation",
532 "total_symbols": 12,
533 "results": [
534 {"kind": "note", "address": "inbox/foo.md", "path": "inbox/foo.md", ...},
535 {"kind": "field", "address": "inbox/foo.md::field:source", ...},
536 {"kind": "section", "address": "inbox/foo.md::section:2:Background", ...}
537 ]
538 }
539
540 Args:
541 symbols: List of :class:`Symbol` objects from :func:`extract_symbols`.
542
543 Returns:
544 JSON-serialisable dict.
545 """
546 return {
547 "domain": "knowtation",
548 "total_symbols": len(symbols),
549 "results": [s.to_dict() for s in symbols],
550 }
File History 2 commits
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6 chore(tests): add docstring to tests/__init__.py so rc14 tr… Human 26 days ago
sha256:d8316ffae901be06347e16ab55be11868eb519dd16ade3e8aa16a99e662f7e62 baseline: rc14 re-baseline after rc3 store corruption recovery Human patch 27 days ago