detector.py
file-level
1
files
1
commits
0
hotspots
0
🧊 dead
0
💥 blast risk
| 1 | """Frontmatter fingerprint detector for the Knowtation domain. |
| 2 | |
| 3 | Determines whether a file or repository belongs to the ``knowtation`` domain |
| 4 | (Markdown vault) or should fall back to ``mist`` (binary artifact store). |
| 5 | |
| 6 | Priority cascade (evaluated in order, first match wins) |
| 7 | ------------------------------------------------------- |
| 8 | **File-level classification** — :func:`classify_note`: |
| 9 | |
| 10 | 1. ``source`` or ``source_type`` key present in frontmatter → ``knowtation``. |
| 11 | These keys identify notes produced by the capture / import pipeline (Telegram, |
| 12 | Slack, PDF, ChatGPT export, …). ``source`` is the SPEC §2.2 canonical name; |
| 13 | ``source_type`` is the import-pipeline alias. Either key is sufficient. |
| 14 | |
| 15 | 2. ``source_id`` key present → ``knowtation``. |
| 16 | Indicates an externally de-duplicated note (message ID, ticket key, etc.). |
| 17 | Presence alone is a strong signal regardless of any other key. |
| 18 | |
| 19 | 3. (``project`` or ``tags``) **AND** ``date`` present → ``knowtation``. |
| 20 | A note with both a project/tag assignment and a creation date is very likely |
| 21 | a structured Knowtation note rather than an unstructured blob. |
| 22 | |
| 23 | 4. Fallback → ``mist``. |
| 24 | Plain prose without any of the above keys, or non-Markdown files, are |
| 25 | treated as mist artifacts. |
| 26 | |
| 27 | **Repository-level classification** — :func:`classify_repo`: |
| 28 | |
| 29 | Checks the above rules in order at the repository level: |
| 30 | |
| 31 | A. ```.knowtation`` marker file exists at the vault root → ``knowtation``. |
| 32 | The marker is an empty (or arbitrary) file that the user or ``muse init`` |
| 33 | places to opt the repo into the knowtation domain explicitly, regardless |
| 34 | of frontmatter coverage. |
| 35 | |
| 36 | B. Frontmatter fingerprint majority vote: walk every ``.md`` file, classify |
| 37 | each one, and return ``knowtation`` when the knowtation fraction meets or |
| 38 | exceeds :data:`REPO_MAJORITY_THRESHOLD` (default 0.5). |
| 39 | |
| 40 | C. Fallback → ``mist``. |
| 41 | |
| 42 | Implementation notes |
| 43 | -------------------- |
| 44 | No external YAML library is used. Frontmatter key detection is done with |
| 45 | a fast regex scan over the raw ``--- ... ---`` block: |
| 46 | |
| 47 | - ``key:`` lines at column 0 (YAML block scalars / mappings). |
| 48 | - Handles single and double quotes around values; ignores values entirely. |
| 49 | - Does **not** parse YAML nested structure — only top-level keys. |
| 50 | |
| 51 | Phase 1.3 will introduce a full typed ``FrontmatterSchema`` with proper YAML |
| 52 | parsing (``pyyaml``). Until then, this detector intentionally stays at the |
| 53 | "key presence" level to remain dependency-free. |
| 54 | """ |
| 55 | |
| 56 | from __future__ import annotations |
| 57 | |
| 58 | import re |
| 59 | import pathlib |
| 60 | import logging |
| 61 | from dataclasses import dataclass, field |
| 62 | from typing import Literal |
| 63 | |
| 64 | logger = logging.getLogger(__name__) |
| 65 | |
| 66 | # --------------------------------------------------------------------------- |
| 67 | # Configuration constants |
| 68 | # --------------------------------------------------------------------------- |
| 69 | |
| 70 | #: Domain name returned when a note / repo is classified as a vault. |
| 71 | DOMAIN_KNOWTATION: Literal["knowtation"] = "knowtation" |
| 72 | |
| 73 | #: Domain name returned when a note / repo is classified as a binary artifact. |
| 74 | DOMAIN_MIST: Literal["mist"] = "mist" |
| 75 | |
| 76 | #: Minimum fraction of Markdown notes that must classify as ``knowtation`` for |
| 77 | #: the whole repository to be classified as ``knowtation`` (rule B). |
| 78 | REPO_MAJORITY_THRESHOLD: float = 0.50 |
| 79 | |
| 80 | #: Name of the optional marker file that unconditionally classifies a repo as |
| 81 | #: ``knowtation`` (rule A). |
| 82 | MARKER_FILENAME: str = ".knowtation" |
| 83 | |
| 84 | #: Markdown note extensions recognised by the detector. |
| 85 | _MARKDOWN_SUFFIXES: frozenset[str] = frozenset({".md", ".markdown", ".mdx"}) |
| 86 | |
| 87 | # --------------------------------------------------------------------------- |
| 88 | # Frontmatter key extractor (regex, no YAML dep) |
| 89 | # --------------------------------------------------------------------------- |
| 90 | |
| 91 | # Match a top-level YAML key: starts at column 0, followed by colon + whitespace |
| 92 | # (or end of line for bare keys). The value is not captured. |
| 93 | _FM_KEY_RE: re.Pattern[str] = re.compile( |
| 94 | r"^([A-Za-z_][A-Za-z0-9_-]*)\s*:", |
| 95 | re.MULTILINE, |
| 96 | ) |
| 97 | |
| 98 | |
| 99 | def extract_frontmatter_keys(content: bytes) -> frozenset[str]: |
| 100 | """Return the set of top-level YAML frontmatter key names in *content*. |
| 101 | |
| 102 | Parses only the leading ``---`` ... ``---`` (or ``---`` ... ``...``) |
| 103 | block. Returns an empty frozenset when no frontmatter is present or when |
| 104 | *content* cannot be decoded as UTF-8. |
| 105 | |
| 106 | **Intentional limitations** (Phase 1.2 scope): |
| 107 | |
| 108 | - Detects only column-0 keys; nested keys (indented) are not captured. |
| 109 | - No YAML value parsing — key *presence* is the only output. |
| 110 | - Does not handle ``%YAML`` directives or multi-document streams. |
| 111 | |
| 112 | Args: |
| 113 | content: Raw bytes of the note file. |
| 114 | |
| 115 | Returns: |
| 116 | Frozenset of key name strings, e.g. ``frozenset({"source", "date", "tags"})``. |
| 117 | Empty frozenset when the file has no frontmatter block. |
| 118 | """ |
| 119 | try: |
| 120 | text = content.decode("utf-8", errors="replace") |
| 121 | except Exception: |
| 122 | return frozenset() |
| 123 | |
| 124 | # Frontmatter must start with '---' on the very first line (no leading |
| 125 | # whitespace or BOM handling needed for Knowtation vault notes). |
| 126 | if not text.startswith("---"): |
| 127 | return frozenset() |
| 128 | |
| 129 | # Find the closing delimiter: '---' or '...' on a line of its own. |
| 130 | rest = text[3:] |
| 131 | end = -1 |
| 132 | for delim in ("\n---", "\n..."): |
| 133 | pos = rest.find(delim) |
| 134 | if pos != -1 and (end == -1 or pos < end): |
| 135 | end = pos |
| 136 | |
| 137 | if end == -1: |
| 138 | return frozenset() |
| 139 | |
| 140 | fm_block = rest[:end] |
| 141 | return frozenset(m.group(1) for m in _FM_KEY_RE.finditer(fm_block)) |
| 142 | |
| 143 | |
| 144 | # --------------------------------------------------------------------------- |
| 145 | # Classification result |
| 146 | # --------------------------------------------------------------------------- |
| 147 | |
| 148 | #: The two domains a note or repo can resolve to. |
| 149 | DomainName = Literal["knowtation", "mist"] |
| 150 | |
| 151 | #: Internal rule labels used in :class:`ClassificationResult`. |
| 152 | _RuleName = Literal[ |
| 153 | "source_key", |
| 154 | "source_id_key", |
| 155 | "project_or_tags_plus_date", |
| 156 | "marker_file", |
| 157 | "majority_vote", |
| 158 | "fallback_mist", |
| 159 | "no_frontmatter", |
| 160 | "no_markdown_notes", |
| 161 | ] |
| 162 | |
| 163 | |
| 164 | @dataclass |
| 165 | class ClassificationResult: |
| 166 | """Result of a note or repository classification. |
| 167 | |
| 168 | ``domain`` |
| 169 | Either ``"knowtation"`` or ``"mist"``. |
| 170 | ``rule`` |
| 171 | The name of the rule that produced this classification. Useful for |
| 172 | debugging and for the ``muse plumbing domain-info`` output. |
| 173 | ``confidence`` |
| 174 | Float in ``[0.0, 1.0]``. Rule 1–4 produce ``1.0``; majority-vote |
| 175 | produces the actual fraction (e.g. ``0.92``); fallback produces |
| 176 | ``0.0``. |
| 177 | ``detail`` |
| 178 | Human-readable explanation. |
| 179 | """ |
| 180 | |
| 181 | domain: DomainName |
| 182 | rule: str |
| 183 | confidence: float = 1.0 |
| 184 | detail: str = "" |
| 185 | |
| 186 | @property |
| 187 | def is_knowtation(self) -> bool: |
| 188 | """``True`` when the domain resolved to ``knowtation``.""" |
| 189 | return self.domain == DOMAIN_KNOWTATION |
| 190 | |
| 191 | |
| 192 | # --------------------------------------------------------------------------- |
| 193 | # File-level classification |
| 194 | # --------------------------------------------------------------------------- |
| 195 | |
| 196 | |
| 197 | def classify_note(content: bytes) -> ClassificationResult: |
| 198 | """Classify one note's raw bytes against the priority cascade. |
| 199 | |
| 200 | Applies rules 1–4 in order and returns the first match. |
| 201 | |
| 202 | Args: |
| 203 | content: Raw bytes of the Markdown (or any) file. |
| 204 | |
| 205 | Returns: |
| 206 | A :class:`ClassificationResult` whose ``domain`` is either |
| 207 | ``"knowtation"`` or ``"mist"``. |
| 208 | |
| 209 | Rule summary:: |
| 210 | |
| 211 | 1. source OR source_type in frontmatter → knowtation |
| 212 | 2. source_id in frontmatter → knowtation |
| 213 | 3. (project OR tags) AND date → knowtation |
| 214 | 4. fallback → mist |
| 215 | """ |
| 216 | keys = extract_frontmatter_keys(content) |
| 217 | |
| 218 | if not keys: |
| 219 | return ClassificationResult( |
| 220 | domain=DOMAIN_MIST, |
| 221 | rule="no_frontmatter", |
| 222 | confidence=0.0, |
| 223 | detail="No YAML frontmatter block found.", |
| 224 | ) |
| 225 | |
| 226 | # Rule 1 — source / source_type key |
| 227 | if "source" in keys or "source_type" in keys: |
| 228 | matched = "source" if "source" in keys else "source_type" |
| 229 | return ClassificationResult( |
| 230 | domain=DOMAIN_KNOWTATION, |
| 231 | rule="source_key", |
| 232 | confidence=1.0, |
| 233 | detail=f"Frontmatter key '{matched}' identifies a capture/import note.", |
| 234 | ) |
| 235 | |
| 236 | # Rule 2 — source_id key |
| 237 | if "source_id" in keys: |
| 238 | return ClassificationResult( |
| 239 | domain=DOMAIN_KNOWTATION, |
| 240 | rule="source_id_key", |
| 241 | confidence=1.0, |
| 242 | detail="Frontmatter key 'source_id' identifies an externally de-duplicated note.", |
| 243 | ) |
| 244 | |
| 245 | # Rule 3 — (project or tags) AND date |
| 246 | has_date = "date" in keys |
| 247 | has_project_or_tags = "project" in keys or "tags" in keys |
| 248 | if has_date and has_project_or_tags: |
| 249 | matched_keys = [k for k in ("project", "tags") if k in keys] |
| 250 | return ClassificationResult( |
| 251 | domain=DOMAIN_KNOWTATION, |
| 252 | rule="project_or_tags_plus_date", |
| 253 | confidence=1.0, |
| 254 | detail=( |
| 255 | f"Frontmatter has date + {matched_keys} — structured Knowtation note." |
| 256 | ), |
| 257 | ) |
| 258 | |
| 259 | # Rule 4 — fallback |
| 260 | return ClassificationResult( |
| 261 | domain=DOMAIN_MIST, |
| 262 | rule="fallback_mist", |
| 263 | confidence=0.0, |
| 264 | detail=( |
| 265 | f"No knowtation-identifying keys found. " |
| 266 | f"Present keys: {sorted(keys) or '(none)'}." |
| 267 | ), |
| 268 | ) |
| 269 | |
| 270 | |
| 271 | # --------------------------------------------------------------------------- |
| 272 | # Repository-level classification |
| 273 | # --------------------------------------------------------------------------- |
| 274 | |
| 275 | |
| 276 | def classify_repo(root: pathlib.Path) -> ClassificationResult: |
| 277 | """Classify a repository as ``knowtation`` or ``mist`` via the repo cascade. |
| 278 | |
| 279 | Rule A: ``.knowtation`` marker file at the vault root → ``knowtation``. |
| 280 | Rule B: Majority vote — walk all ``.md`` files, classify each, return |
| 281 | ``knowtation`` when the knowtation fraction ≥ :data:`REPO_MAJORITY_THRESHOLD`. |
| 282 | Rule C: Fallback → ``mist``. |
| 283 | |
| 284 | Args: |
| 285 | root: Vault root directory path. |
| 286 | |
| 287 | Returns: |
| 288 | A :class:`ClassificationResult` for the repository. |
| 289 | """ |
| 290 | # Rule A — explicit marker file |
| 291 | marker = root / MARKER_FILENAME |
| 292 | if marker.is_file(): |
| 293 | return ClassificationResult( |
| 294 | domain=DOMAIN_KNOWTATION, |
| 295 | rule="marker_file", |
| 296 | confidence=1.0, |
| 297 | detail=f"Marker file '{MARKER_FILENAME}' found at vault root.", |
| 298 | ) |
| 299 | |
| 300 | # Rule B — frontmatter majority vote across all Markdown files |
| 301 | total = 0 |
| 302 | knowtation_count = 0 |
| 303 | for md_path in _iter_markdown_files(root): |
| 304 | total += 1 |
| 305 | try: |
| 306 | content = md_path.read_bytes() |
| 307 | except OSError: |
| 308 | continue |
| 309 | result = classify_note(content) |
| 310 | if result.is_knowtation: |
| 311 | knowtation_count += 1 |
| 312 | |
| 313 | if total == 0: |
| 314 | return ClassificationResult( |
| 315 | domain=DOMAIN_MIST, |
| 316 | rule="no_markdown_notes", |
| 317 | confidence=0.0, |
| 318 | detail="No Markdown files found in repository.", |
| 319 | ) |
| 320 | |
| 321 | fraction = knowtation_count / total |
| 322 | if fraction >= REPO_MAJORITY_THRESHOLD: |
| 323 | return ClassificationResult( |
| 324 | domain=DOMAIN_KNOWTATION, |
| 325 | rule="majority_vote", |
| 326 | confidence=fraction, |
| 327 | detail=( |
| 328 | f"{knowtation_count}/{total} Markdown notes classified as knowtation " |
| 329 | f"({fraction:.1%} ≥ threshold {REPO_MAJORITY_THRESHOLD:.0%})." |
| 330 | ), |
| 331 | ) |
| 332 | |
| 333 | # Rule C — fallback |
| 334 | return ClassificationResult( |
| 335 | domain=DOMAIN_MIST, |
| 336 | rule="fallback_mist", |
| 337 | confidence=1.0 - fraction, |
| 338 | detail=( |
| 339 | f"Only {knowtation_count}/{total} notes classified as knowtation " |
| 340 | f"({fraction:.1%} < threshold {REPO_MAJORITY_THRESHOLD:.0%})." |
| 341 | ), |
| 342 | ) |
| 343 | |
| 344 | |
| 345 | def _iter_markdown_files(root: pathlib.Path) -> list[pathlib.Path]: |
| 346 | """Return all Markdown files under *root*, skipping hidden directories. |
| 347 | |
| 348 | Ignores ``.git``, ``.muse``, ``.obsidian``, ``.knowtation``, and other |
| 349 | dot-directories that should never contain vault notes. |
| 350 | |
| 351 | Args: |
| 352 | root: Directory to walk. |
| 353 | |
| 354 | Returns: |
| 355 | Sorted list of :class:`pathlib.Path` objects for each ``.md`` / |
| 356 | ``.markdown`` / ``.mdx`` file found. |
| 357 | """ |
| 358 | from muse.plugins.knowtation.plugin import _ALWAYS_IGNORE_DIRS |
| 359 | |
| 360 | results: list[pathlib.Path] = [] |
| 361 | import os |
| 362 | |
| 363 | root_str = str(root) |
| 364 | for dirpath, dirnames, filenames in os.walk(root_str, followlinks=False): |
| 365 | # Prune directories we should not descend into. |
| 366 | dirnames[:] = sorted( |
| 367 | d for d in dirnames |
| 368 | if d not in _ALWAYS_IGNORE_DIRS |
| 369 | and not d.startswith(".") |
| 370 | ) |
| 371 | for fname in sorted(filenames): |
| 372 | suffix = pathlib.PurePosixPath(fname).suffix.lower() |
| 373 | if suffix in _MARKDOWN_SUFFIXES: |
| 374 | results.append(pathlib.Path(dirpath) / fname) |
| 375 | |
| 376 | return sorted(results) |
| 377 | |
| 378 | |
| 379 | # --------------------------------------------------------------------------- |
| 380 | # Batch classification helpers |
| 381 | # --------------------------------------------------------------------------- |
| 382 | |
| 383 | |
| 384 | def classify_many( |
| 385 | notes: dict[str, bytes], |
| 386 | ) -> dict[str, ClassificationResult]: |
| 387 | """Classify multiple notes in one call. |
| 388 | |
| 389 | Args: |
| 390 | notes: Mapping of ``{label: content_bytes}``. The label is typically |
| 391 | a vault-relative file path. |
| 392 | |
| 393 | Returns: |
| 394 | Mapping of ``{label: ClassificationResult}``. |
| 395 | """ |
| 396 | return {label: classify_note(content) for label, content in notes.items()} |
| 397 | |
| 398 | |
| 399 | def knowtation_fraction(notes: dict[str, bytes]) -> float: |
| 400 | """Return the fraction of *notes* that classify as ``knowtation``. |
| 401 | |
| 402 | Args: |
| 403 | notes: Mapping of ``{label: content_bytes}``. |
| 404 | |
| 405 | Returns: |
| 406 | Float in ``[0.0, 1.0]``. Returns ``0.0`` when *notes* is empty. |
| 407 | """ |
| 408 | if not notes: |
| 409 | return 0.0 |
| 410 | results = classify_many(notes) |
| 411 | count = sum(1 for r in results.values() if r.is_knowtation) |
| 412 | return count / len(notes) |