"""Knowtation symbol model — note / field / section address extraction. This module implements the three symbol kinds that Muse tracks for Knowtation vault notes. Symbol addresses follow the same ``file::symbol`` convention used by the code domain (``src/utils.py::calculate_total``) so that generic Muse commands (``muse code symbols``, ``muse code impact``, ``muse code blame``) can be made domain-aware in a later phase without changing their output schema. Symbol kinds ------------ ``note`` The note itself. Address = vault-relative path (e.g. ``inbox/foo.md``). There is exactly one ``NoteSymbol`` per note. ``field`` A frontmatter key-value pair. Address = ``{path}::field:{key}`` (e.g. ``inbox/foo.md::field:source``). One ``FieldSymbol`` per non-empty field in the parsed frontmatter. ``section`` A Markdown heading and the body text beneath it. Address = ``{path}::section:{level}:{title}`` (e.g. ``inbox/foo.md::section:2:Background``). Level is the number of ``#`` characters (1–6). A special ``level=0`` section (title ``"(preamble)"``) is generated for body text that appears before the first heading. Section splitter ---------------- Deterministic Python port of the ``splitByHeadingOrSize`` heading regex from ``knowtation/lib/chunk.mjs``. The original JS regex is:: /(?=^#{2,3}\\s)/m This module generalises to all heading levels (H1–H6) for the symbol model (``chunk.mjs`` only uses H2/H3 for chunking). The port does **not** call Node.js — it uses :mod:`re` with ``re.MULTILINE``. Address conventions ------------------- - Separator between file and symbol: ``::`` - ``field`` prefix + colon: ``field:`` - ``section`` prefix + colon: ``section:`` - Level immediately follows ``section:``: ``section:2:Background`` - All components are left unescaped so addresses are human-readable in logs. """ from __future__ import annotations import re from dataclasses import dataclass, field from typing import Any, Literal from muse.plugins.knowtation.parser import ParsedFrontmatter, parse_frontmatter # ────────────────────────────────────────────────────────────────────────────── # Address helpers # ────────────────────────────────────────────────────────────────────────────── #: Separator between vault-relative path and the symbol qualifier. ADDRESS_SEP: str = "::" #: Prefix for frontmatter field addresses (after the ``::`` separator). FIELD_PREFIX: str = "field" #: Prefix for section addresses (after the ``::`` separator). SECTION_PREFIX: str = "section" def make_note_address(path: str) -> str: """Return the address for a note symbol (the vault-relative path itself). Args: path: Vault-relative file path (e.g. ``"inbox/foo.md"``). Returns: The same path string used as the note's canonical address. """ return path def make_field_address(path: str, key: str) -> str: """Return the address for a frontmatter field symbol. Args: path: Vault-relative file path. key: Frontmatter key name (e.g. ``"source"``). Returns: Address string, e.g. ``"inbox/foo.md::field:source"``. """ return f"{path}{ADDRESS_SEP}{FIELD_PREFIX}:{key}" def make_section_address(path: str, level: int, title: str) -> str: """Return the address for a section symbol. Args: path: Vault-relative file path. level: Heading level (0 = preamble, 1–6 = H1–H6). title: Heading text without the ``#`` characters. Returns: Address string, e.g. ``"inbox/foo.md::section:2:Background"``. """ return f"{path}{ADDRESS_SEP}{SECTION_PREFIX}:{level}:{title}" def parse_address(address: str) -> dict[str, str]: """Decompose a symbol address into its components. Args: address: A full symbol address string. Returns: Dict with keys ``"path"``, ``"kind"``, and (for field/section) their qualifier keys. ``"kind"`` is ``"note"``, ``"field"``, or ``"section"``. Examples:: parse_address("inbox/foo.md") → {"path": "inbox/foo.md", "kind": "note"} parse_address("inbox/foo.md::field:source") → {"path": "inbox/foo.md", "kind": "field", "key": "source"} parse_address("inbox/foo.md::section:2:Background") → {"path": "inbox/foo.md", "kind": "section", "level": "2", "title": "Background"} """ if ADDRESS_SEP not in address: return {"path": address, "kind": "note"} path, qualifier = address.split(ADDRESS_SEP, 1) if qualifier.startswith(f"{FIELD_PREFIX}:"): key = qualifier[len(FIELD_PREFIX) + 1:] return {"path": path, "kind": "field", "key": key} if qualifier.startswith(f"{SECTION_PREFIX}:"): rest = qualifier[len(SECTION_PREFIX) + 1:] level_str, _, title = rest.partition(":") return {"path": path, "kind": "section", "level": level_str, "title": title} return {"path": path, "kind": "unknown", "qualifier": qualifier} # ────────────────────────────────────────────────────────────────────────────── # Symbol dataclasses # ────────────────────────────────────────────────────────────────────────────── @dataclass class NoteSymbol: """Top-level symbol representing the vault note as a whole. There is exactly one ``NoteSymbol`` per note, always the first entry returned by :func:`extract_symbols`. """ kind: Literal["note"] = field(default="note", init=False) path: str = "" address: str = "" #: Display title — from frontmatter ``title`` field, first H1 heading, or filename. title: str = "" #: Project slug from frontmatter (``None`` when absent). project: str | None = None #: Tags from frontmatter (empty list when absent). tags: list[str] = field(default_factory=list) def to_dict(self) -> dict[str, Any]: return { "kind": self.kind, "address": self.address, "path": self.path, "title": self.title, "project": self.project, "tags": self.tags, } @dataclass class FieldSymbol: """Symbol for a single frontmatter key-value pair. One ``FieldSymbol`` is emitted for every non-``None`` / non-empty field in the parsed frontmatter, including ``extra`` keys. """ kind: Literal["field"] = field(default="field", init=False) path: str = "" key: str = "" value: Any = None #: Python type name of the value (``"str"``, ``"list"``, ``"bool"``, …). value_type: str = "" address: str = "" def to_dict(self) -> dict[str, Any]: return { "kind": self.kind, "address": self.address, "path": self.path, "key": self.key, "value_type": self.value_type, } @dataclass class SectionSymbol: """Symbol for one Markdown heading section. The body text of each section runs from (and including) the heading line up to (but not including) the next heading of any level, or end-of-note. A ``level=0`` section with ``title="(preamble)"`` is generated for body text that appears before the first heading. """ kind: Literal["section"] = field(default="section", init=False) path: str = "" #: Heading level: 0 = preamble, 1–6 = H1–H6. level: int = 0 #: Heading text stripped of ``#`` characters and surrounding whitespace. title: str = "" #: 0-based index among all sections in this note (preamble = index 0 if present). index: int = 0 #: 1-based line number of the heading within the note body (not counting frontmatter). line: int = 1 #: Full section text (heading line + body until next heading). body: str = "" address: str = "" def to_dict(self) -> dict[str, Any]: return { "kind": self.kind, "address": self.address, "path": self.path, "level": self.level, "title": self.title, "index": self.index, "line": self.line, } #: Union type for all symbol kinds. Symbol = NoteSymbol | FieldSymbol | SectionSymbol # ────────────────────────────────────────────────────────────────────────────── # Section splitter — deterministic Python port of chunk.mjs heading regex # ────────────────────────────────────────────────────────────────────────────── # Original JS (chunk.mjs): /(?=^#{2,3}\s)/m — lookahead on ## or ### # Extended to H1–H6 for the full symbol model. # ATX heading: optional closing hashes stripped (CommonMark §4.2 closed ATX). # (?:#+\s*)? at the end removes patterns like "## Title ##". _HEADING_RE: re.Pattern[str] = re.compile( r"^(#{1,6})\s+(.*?)\s*(?:#+\s*)?$", re.MULTILINE ) #: Markdown extensions recognised as note files. _NOTE_EXTENSIONS: frozenset[str] = frozenset({".md", ".markdown", ".mdx"}) def _strip_frontmatter(text: str) -> tuple[str, int]: """Return (body_text, body_start_line) with frontmatter removed. ``body_start_line`` is the 1-based line number in *text* where the body begins (used to offset section line numbers so they refer to the raw file). Args: text: Full note text decoded from bytes. Returns: Tuple of (body_without_frontmatter, 1-based_line_offset). """ if not text.startswith("---"): return text, 1 rest = text[3:] end = -1 for delim in ("\n---", "\n..."): pos = rest.find(delim) if pos != -1 and (end == -1 or pos < end): end = pos if end == -1: return text, 1 # Skip past the closing delimiter line close_end = end + len("\n---") if close_end < len(rest) and rest[close_end] == "\n": close_end += 1 body = rest[close_end:] # Count lines consumed by the frontmatter block fm_text = text[: 3 + close_end] fm_lines = fm_text.count("\n") return body, fm_lines + 1 def split_sections(body: str) -> list[tuple[int, str, str, int]]: """Split a Markdown body into sections by heading. Deterministic Python port of ``splitByHeadingOrSize`` from ``knowtation/lib/chunk.mjs``, extended from H2/H3 to H1–H6 for symbol granularity. A ``(0, "(preamble)", text, 1)`` entry is prepended when text exists before the first heading. Args: body: Markdown body text with frontmatter already stripped. Returns: List of ``(level, title, section_body, 1_based_line)`` tuples. Empty list when *body* is blank. """ if not body.strip(): return [] headings = list(_HEADING_RE.finditer(body)) results: list[tuple[int, str, str, int]] = [] # Preamble — text before the first heading preamble_end = headings[0].start() if headings else len(body) preamble = body[:preamble_end] if preamble.strip(): results.append((0, "(preamble)", preamble, 1)) for i, match in enumerate(headings): level = len(match.group(1)) title = match.group(2) start = match.start() end = headings[i + 1].start() if i + 1 < len(headings) else len(body) section_body = body[start:end] # Line number within the body (1-based) body_line = body[:start].count("\n") + 1 results.append((level, title, section_body, body_line)) return results # ────────────────────────────────────────────────────────────────────────────── # Field extraction # ────────────────────────────────────────────────────────────────────────────── #: Frontmatter fields to emit as FieldSymbols. All known non-empty fields #: from ParsedFrontmatter.to_dict() plus extra keys. def _field_symbols_from_frontmatter( path: str, fm: ParsedFrontmatter ) -> list[FieldSymbol]: """Return one :class:`FieldSymbol` per non-empty frontmatter field. Args: path: Vault-relative note path. fm: Parsed frontmatter from :func:`~parser.parse_frontmatter`. Returns: List of :class:`FieldSymbol` instances in SPEC §2 field order. """ symbols: list[FieldSymbol] = [] for key, value in fm.to_dict().items(): if value is None: continue if isinstance(value, list) and not value: continue if isinstance(value, bool) and not value: continue value_type = type(value).__name__ symbols.append( FieldSymbol( path=path, key=key, value=value, value_type=value_type, address=make_field_address(path, key), ) ) return symbols # ────────────────────────────────────────────────────────────────────────────── # Note title resolution # ────────────────────────────────────────────────────────────────────────────── def _resolve_note_title( path: str, fm: ParsedFrontmatter | None, body: str ) -> str: """Resolve the display title for a note. Priority: 1. ``title`` field in frontmatter (SPEC §2.1). 2. First H1 heading in the body. 3. Filename stem (without extension). Args: path: Vault-relative note path. fm: Parsed frontmatter (or ``None`` when absent). body: Note body with frontmatter stripped. Returns: Display title string. """ if fm is not None and fm.title: return fm.title # First H1 heading h1 = _HEADING_RE.search(body) if h1 and len(h1.group(1)) == 1: return h1.group(2) # Filename stem stem = path.rsplit("/", 1)[-1] for ext in _NOTE_EXTENSIONS: if stem.endswith(ext): stem = stem[: -len(ext)] break return stem # ────────────────────────────────────────────────────────────────────────────── # Main extraction API # ────────────────────────────────────────────────────────────────────────────── def extract_symbols(path: str, content: bytes) -> list[Symbol]: """Extract all symbols from a vault note's raw bytes. Returns symbols in this order: 1. One :class:`NoteSymbol` (always present). 2. One :class:`FieldSymbol` per non-empty frontmatter field. 3. One :class:`SectionSymbol` per heading section (including preamble when present). Args: path: Vault-relative file path (e.g. ``"inbox/foo.md"``). content: Raw bytes of the note file. Returns: List of :class:`Symbol` objects. Never raises — returns only the ``NoteSymbol`` when parsing or splitting fails. """ symbols: list[Symbol] = [] # ── Parse ──────────────────────────────────────────────────────────────── try: text = content.decode("utf-8", errors="replace") except Exception: text = "" fm = parse_frontmatter(content) body, _body_line_offset = _strip_frontmatter(text) # ── NoteSymbol ─────────────────────────────────────────────────────────── note_title = _resolve_note_title(path, fm, body) note_sym = NoteSymbol( path=path, address=make_note_address(path), title=note_title, project=fm.project if fm else None, tags=list(fm.tags) if fm else [], ) symbols.append(note_sym) # ── FieldSymbols ───────────────────────────────────────────────────────── if fm is not None: symbols.extend(_field_symbols_from_frontmatter(path, fm)) # ── SectionSymbols ─────────────────────────────────────────────────────── try: sections = split_sections(body) except Exception: sections = [] for idx, (level, title, sec_body, body_line) in enumerate(sections): symbols.append( SectionSymbol( path=path, level=level, title=title, index=idx, line=body_line, body=sec_body, address=make_section_address(path, level, title), ) ) return symbols def extract_sections(path: str, content: bytes) -> list[SectionSymbol]: """Return only the :class:`SectionSymbol` entries for a note. Convenience wrapper around :func:`extract_symbols` for callers that only need the section graph (e.g. diff and merge engines). Args: path: Vault-relative file path. content: Raw bytes of the note file. Returns: List of :class:`SectionSymbol` instances in document order. """ return [s for s in extract_symbols(path, content) if isinstance(s, SectionSymbol)] def extract_fields(path: str, content: bytes) -> list[FieldSymbol]: """Return only the :class:`FieldSymbol` entries for a note. Args: path: Vault-relative file path. content: Raw bytes of the note file. Returns: List of :class:`FieldSymbol` instances in SPEC §2 field order. """ return [s for s in extract_symbols(path, content) if isinstance(s, FieldSymbol)] def symbols_to_json(symbols: list[Symbol]) -> dict[str, Any]: """Serialise a symbol list to the ``muse code symbols --json`` output shape. The schema matches the ``code`` domain's JSON output so the same tooling can consume vault symbols without special-casing: .. code-block:: json { "domain": "knowtation", "total_symbols": 12, "results": [ {"kind": "note", "address": "inbox/foo.md", "path": "inbox/foo.md", ...}, {"kind": "field", "address": "inbox/foo.md::field:source", ...}, {"kind": "section", "address": "inbox/foo.md::section:2:Background", ...} ] } Args: symbols: List of :class:`Symbol` objects from :func:`extract_symbols`. Returns: JSON-serialisable dict. """ return { "domain": "knowtation", "total_symbols": len(symbols), "results": [s.to_dict() for s in symbols], }