"""Frontmatter fingerprint detector for the Knowtation domain. Determines whether a file or repository belongs to the ``knowtation`` domain (Markdown vault) or should fall back to ``mist`` (binary artifact store). Priority cascade (evaluated in order, first match wins) ------------------------------------------------------- **File-level classification** — :func:`classify_note`: 1. ``source`` or ``source_type`` key present in frontmatter → ``knowtation``. These keys identify notes produced by the capture / import pipeline (Telegram, Slack, PDF, ChatGPT export, …). ``source`` is the SPEC §2.2 canonical name; ``source_type`` is the import-pipeline alias. Either key is sufficient. 2. ``source_id`` key present → ``knowtation``. Indicates an externally de-duplicated note (message ID, ticket key, etc.). Presence alone is a strong signal regardless of any other key. 3. (``project`` or ``tags``) **AND** ``date`` present → ``knowtation``. A note with both a project/tag assignment and a creation date is very likely a structured Knowtation note rather than an unstructured blob. 4. Fallback → ``mist``. Plain prose without any of the above keys, or non-Markdown files, are treated as mist artifacts. **Repository-level classification** — :func:`classify_repo`: Checks the above rules in order at the repository level: A. ```.knowtation`` marker file exists at the vault root → ``knowtation``. The marker is an empty (or arbitrary) file that the user or ``muse init`` places to opt the repo into the knowtation domain explicitly, regardless of frontmatter coverage. B. Frontmatter fingerprint majority vote: walk every ``.md`` file, classify each one, and return ``knowtation`` when the knowtation fraction meets or exceeds :data:`REPO_MAJORITY_THRESHOLD` (default 0.5). C. Fallback → ``mist``. Implementation notes -------------------- No external YAML library is used. Frontmatter key detection is done with a fast regex scan over the raw ``--- ... ---`` block: - ``key:`` lines at column 0 (YAML block scalars / mappings). - Handles single and double quotes around values; ignores values entirely. - Does **not** parse YAML nested structure — only top-level keys. Phase 1.3 will introduce a full typed ``FrontmatterSchema`` with proper YAML parsing (``pyyaml``). Until then, this detector intentionally stays at the "key presence" level to remain dependency-free. """ from __future__ import annotations import re import pathlib import logging from dataclasses import dataclass, field from typing import Literal logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- # Configuration constants # --------------------------------------------------------------------------- #: Domain name returned when a note / repo is classified as a vault. DOMAIN_KNOWTATION: Literal["knowtation"] = "knowtation" #: Domain name returned when a note / repo is classified as a binary artifact. DOMAIN_MIST: Literal["mist"] = "mist" #: Minimum fraction of Markdown notes that must classify as ``knowtation`` for #: the whole repository to be classified as ``knowtation`` (rule B). REPO_MAJORITY_THRESHOLD: float = 0.50 #: Name of the optional marker file that unconditionally classifies a repo as #: ``knowtation`` (rule A). MARKER_FILENAME: str = ".knowtation" #: Markdown note extensions recognised by the detector. _MARKDOWN_SUFFIXES: frozenset[str] = frozenset({".md", ".markdown", ".mdx"}) # --------------------------------------------------------------------------- # Frontmatter key extractor (regex, no YAML dep) # --------------------------------------------------------------------------- # Match a top-level YAML key: starts at column 0, followed by colon + whitespace # (or end of line for bare keys). The value is not captured. _FM_KEY_RE: re.Pattern[str] = re.compile( r"^([A-Za-z_][A-Za-z0-9_-]*)\s*:", re.MULTILINE, ) def extract_frontmatter_keys(content: bytes) -> frozenset[str]: """Return the set of top-level YAML frontmatter key names in *content*. Parses only the leading ``---`` ... ``---`` (or ``---`` ... ``...``) block. Returns an empty frozenset when no frontmatter is present or when *content* cannot be decoded as UTF-8. **Intentional limitations** (Phase 1.2 scope): - Detects only column-0 keys; nested keys (indented) are not captured. - No YAML value parsing — key *presence* is the only output. - Does not handle ``%YAML`` directives or multi-document streams. Args: content: Raw bytes of the note file. Returns: Frozenset of key name strings, e.g. ``frozenset({"source", "date", "tags"})``. Empty frozenset when the file has no frontmatter block. """ try: text = content.decode("utf-8", errors="replace") except Exception: return frozenset() # Frontmatter must start with '---' on the very first line (no leading # whitespace or BOM handling needed for Knowtation vault notes). if not text.startswith("---"): return frozenset() # Find the closing delimiter: '---' or '...' on a line of its own. rest = text[3:] end = -1 for delim in ("\n---", "\n..."): pos = rest.find(delim) if pos != -1 and (end == -1 or pos < end): end = pos if end == -1: return frozenset() fm_block = rest[:end] return frozenset(m.group(1) for m in _FM_KEY_RE.finditer(fm_block)) # --------------------------------------------------------------------------- # Classification result # --------------------------------------------------------------------------- #: The two domains a note or repo can resolve to. DomainName = Literal["knowtation", "mist"] #: Internal rule labels used in :class:`ClassificationResult`. _RuleName = Literal[ "source_key", "source_id_key", "project_or_tags_plus_date", "marker_file", "majority_vote", "fallback_mist", "no_frontmatter", "no_markdown_notes", ] @dataclass class ClassificationResult: """Result of a note or repository classification. ``domain`` Either ``"knowtation"`` or ``"mist"``. ``rule`` The name of the rule that produced this classification. Useful for debugging and for the ``muse plumbing domain-info`` output. ``confidence`` Float in ``[0.0, 1.0]``. Rule 1–4 produce ``1.0``; majority-vote produces the actual fraction (e.g. ``0.92``); fallback produces ``0.0``. ``detail`` Human-readable explanation. """ domain: DomainName rule: str confidence: float = 1.0 detail: str = "" @property def is_knowtation(self) -> bool: """``True`` when the domain resolved to ``knowtation``.""" return self.domain == DOMAIN_KNOWTATION # --------------------------------------------------------------------------- # File-level classification # --------------------------------------------------------------------------- def classify_note(content: bytes) -> ClassificationResult: """Classify one note's raw bytes against the priority cascade. Applies rules 1–4 in order and returns the first match. Args: content: Raw bytes of the Markdown (or any) file. Returns: A :class:`ClassificationResult` whose ``domain`` is either ``"knowtation"`` or ``"mist"``. Rule summary:: 1. source OR source_type in frontmatter → knowtation 2. source_id in frontmatter → knowtation 3. (project OR tags) AND date → knowtation 4. fallback → mist """ keys = extract_frontmatter_keys(content) if not keys: return ClassificationResult( domain=DOMAIN_MIST, rule="no_frontmatter", confidence=0.0, detail="No YAML frontmatter block found.", ) # Rule 1 — source / source_type key if "source" in keys or "source_type" in keys: matched = "source" if "source" in keys else "source_type" return ClassificationResult( domain=DOMAIN_KNOWTATION, rule="source_key", confidence=1.0, detail=f"Frontmatter key '{matched}' identifies a capture/import note.", ) # Rule 2 — source_id key if "source_id" in keys: return ClassificationResult( domain=DOMAIN_KNOWTATION, rule="source_id_key", confidence=1.0, detail="Frontmatter key 'source_id' identifies an externally de-duplicated note.", ) # Rule 3 — (project or tags) AND date has_date = "date" in keys has_project_or_tags = "project" in keys or "tags" in keys if has_date and has_project_or_tags: matched_keys = [k for k in ("project", "tags") if k in keys] return ClassificationResult( domain=DOMAIN_KNOWTATION, rule="project_or_tags_plus_date", confidence=1.0, detail=( f"Frontmatter has date + {matched_keys} — structured Knowtation note." ), ) # Rule 4 — fallback return ClassificationResult( domain=DOMAIN_MIST, rule="fallback_mist", confidence=0.0, detail=( f"No knowtation-identifying keys found. " f"Present keys: {sorted(keys) or '(none)'}." ), ) # --------------------------------------------------------------------------- # Repository-level classification # --------------------------------------------------------------------------- def classify_repo(root: pathlib.Path) -> ClassificationResult: """Classify a repository as ``knowtation`` or ``mist`` via the repo cascade. Rule A: ``.knowtation`` marker file at the vault root → ``knowtation``. Rule B: Majority vote — walk all ``.md`` files, classify each, return ``knowtation`` when the knowtation fraction ≥ :data:`REPO_MAJORITY_THRESHOLD`. Rule C: Fallback → ``mist``. Args: root: Vault root directory path. Returns: A :class:`ClassificationResult` for the repository. """ # Rule A — explicit marker file marker = root / MARKER_FILENAME if marker.is_file(): return ClassificationResult( domain=DOMAIN_KNOWTATION, rule="marker_file", confidence=1.0, detail=f"Marker file '{MARKER_FILENAME}' found at vault root.", ) # Rule B — frontmatter majority vote across all Markdown files total = 0 knowtation_count = 0 for md_path in _iter_markdown_files(root): total += 1 try: content = md_path.read_bytes() except OSError: continue result = classify_note(content) if result.is_knowtation: knowtation_count += 1 if total == 0: return ClassificationResult( domain=DOMAIN_MIST, rule="no_markdown_notes", confidence=0.0, detail="No Markdown files found in repository.", ) fraction = knowtation_count / total if fraction >= REPO_MAJORITY_THRESHOLD: return ClassificationResult( domain=DOMAIN_KNOWTATION, rule="majority_vote", confidence=fraction, detail=( f"{knowtation_count}/{total} Markdown notes classified as knowtation " f"({fraction:.1%} ≥ threshold {REPO_MAJORITY_THRESHOLD:.0%})." ), ) # Rule C — fallback return ClassificationResult( domain=DOMAIN_MIST, rule="fallback_mist", confidence=1.0 - fraction, detail=( f"Only {knowtation_count}/{total} notes classified as knowtation " f"({fraction:.1%} < threshold {REPO_MAJORITY_THRESHOLD:.0%})." ), ) def _iter_markdown_files(root: pathlib.Path) -> list[pathlib.Path]: """Return all Markdown files under *root*, skipping hidden directories. Ignores ``.git``, ``.muse``, ``.obsidian``, ``.knowtation``, and other dot-directories that should never contain vault notes. Args: root: Directory to walk. Returns: Sorted list of :class:`pathlib.Path` objects for each ``.md`` / ``.markdown`` / ``.mdx`` file found. """ from muse.plugins.knowtation.plugin import _ALWAYS_IGNORE_DIRS results: list[pathlib.Path] = [] import os root_str = str(root) for dirpath, dirnames, filenames in os.walk(root_str, followlinks=False): # Prune directories we should not descend into. dirnames[:] = sorted( d for d in dirnames if d not in _ALWAYS_IGNORE_DIRS and not d.startswith(".") ) for fname in sorted(filenames): suffix = pathlib.PurePosixPath(fname).suffix.lower() if suffix in _MARKDOWN_SUFFIXES: results.append(pathlib.Path(dirpath) / fname) return sorted(results) # --------------------------------------------------------------------------- # Batch classification helpers # --------------------------------------------------------------------------- def classify_many( notes: dict[str, bytes], ) -> dict[str, ClassificationResult]: """Classify multiple notes in one call. Args: notes: Mapping of ``{label: content_bytes}``. The label is typically a vault-relative file path. Returns: Mapping of ``{label: ClassificationResult}``. """ return {label: classify_note(content) for label, content in notes.items()} def knowtation_fraction(notes: dict[str, bytes]) -> float: """Return the fraction of *notes* that classify as ``knowtation``. Args: notes: Mapping of ``{label: content_bytes}``. Returns: Float in ``[0.0, 1.0]``. Returns ``0.0`` when *notes* is empty. """ if not notes: return 0.0 results = classify_many(notes) count = sum(1 for r in results.values() if r.is_knowtation) return count / len(notes)