"""Tests for Phase 3.1 — Knowtation link indexer. Test tiers covered (Rule #0): 1. Unit — wikilink extraction (50 link-shape fixtures) 2. Unit — markdown .md link extraction 3. Unit — path resolution semantics (relative, absolute, escape attempts) 4. Unit — frontmatter strip semantics 5. Integration — build_link_index on synthetic vaults (broken links, entity, case sensitivity, duplicates) 6. Data-integrity — round-trip on the real Knowtation vault when present 7. Performance — build < 10s on a 5k-note synthetic vault 8. Stress — large notes do not OOM; oversized notes are skipped 9. Security — path traversal in markdown links, deeply nested directories, binary content, NUL bytes """ from __future__ import annotations import os import pathlib import time import pytest from muse.plugins.knowtation.link_index import ( LinkIndex, ResolvedLink, _normalise_rel_path, _strip_frontmatter, _vault_basename_key, _wikilink_target_key, build_link_index, extract_md_links, extract_wikilinks, ) # ============================================================================ # Helpers # ============================================================================ def _note(*, fm: str = "", body: str = "") -> bytes: if fm: return f"---\n{fm}\n---\n{body}".encode() return body.encode() def _make_vault(tmp_path: pathlib.Path, files: dict[str, bytes | str]) -> pathlib.Path: """Populate *tmp_path* with the given files and return the vault root.""" for rel, content in files.items(): full = tmp_path / rel full.parent.mkdir(parents=True, exist_ok=True) if isinstance(content, str): full.write_text(content, encoding="utf-8") else: full.write_bytes(content) return tmp_path # ============================================================================ # TestWikilinkTargetKey (parity with lib/wikilink.mjs) # ============================================================================ class TestWikilinkTargetKey: def test_plain_name(self) -> None: assert _wikilink_target_key("My Note") == "my note" def test_strips_md_extension(self) -> None: assert _wikilink_target_key("My Note.md") == "my note" def test_strips_md_extension_uppercase(self) -> None: assert _wikilink_target_key("My Note.MD") == "my note" def test_takes_last_path_segment(self) -> None: assert _wikilink_target_key("folder/sub/Note") == "note" def test_handles_windows_separators(self) -> None: assert _wikilink_target_key("folder\\sub\\Note") == "note" def test_trims_whitespace(self) -> None: assert _wikilink_target_key(" Note ") == "note" def test_empty_string(self) -> None: assert _wikilink_target_key("") == "" def test_only_whitespace(self) -> None: assert _wikilink_target_key(" ") == "" class TestVaultBasenameKey: def test_root_file(self) -> None: assert _vault_basename_key("Note.md") == "note" def test_nested_file(self) -> None: assert _vault_basename_key("inbox/captures/Note.md") == "note" def test_no_extension(self) -> None: assert _vault_basename_key("inbox/Note") == "note" # ============================================================================ # TestExtractWikilinks (50 link-shape fixtures via parametrise) # ============================================================================ _WIKILINK_FIXTURES: list[tuple[str, list[tuple[str, str, str]]]] = [ # (body, expected_list_of_(target,fragment,alias)) ("[[Note]]", [("Note", "", "")]), ("[[my note]]", [("my note", "", "")]), ("[[folder/Note]]", [("folder/Note", "", "")]), ("[[Note|Display Text]]", [("Note", "", "Display Text")]), ("[[Note#Heading]]", [("Note", "Heading", "")]), ("[[Note#Heading|Alias]]", [("Note", "Heading", "Alias")]), ("[[a]] and [[b]]", [("a", "", ""), ("b", "", "")]), ("no links here", []), ("", []), ("[[]]", []), # empty wikilink invalid ("[[ ]]", [("", "", "")]), # whitespace-only — extractor strips to empty ("[Markdown](url)", []), # not a wikilink ("[[Note.md]]", [("Note.md", "", "")]), ("[[Note With Spaces]]", [("Note With Spaces", "", "")]), ("[[Note]] [[Note]]", [("Note", "", ""), ("Note", "", "")]), ("[[CamelCase]]", [("CamelCase", "", "")]), ("[[note-with-hyphens]]", [("note-with-hyphens", "", "")]), ("[[note_with_underscores]]", [("note_with_underscores", "", "")]), ("[[2025-01-15-daily]]", [("2025-01-15-daily", "", "")]), ("text [[link]] more text", [("link", "", "")]), ("a\n[[link]]\nb", [("link", "", "")]), ("[[a/b/c]]", [("a/b/c", "", "")]), ("[[note#section with spaces]]", [("note", "section with spaces", "")]), ("[[note|alias with spaces]]", [("note", "", "alias with spaces")]), ("[Not a wikilink]", []), ("[Not a wikilink](url)", []), ("[[a]]b[[c]]", [("a", "", ""), ("c", "", "")]), ("[[日本語]]", [("日本語", "", "")]), ("[[émoji-🚀]]", [("émoji-🚀", "", "")]), ("[[note|alias|extra]]", [("note", "", "alias|extra")]), ("[[a]][[b]][[c]]", [("a", "", ""), ("b", "", ""), ("c", "", "")]), ("[[note ]] [[ note]]", [("note", "", ""), ("note", "", "")]), ("```\n[[in code block]]\n```", [("in code block", "", "")]), # body parser doesn't strip code ("[[A]]\n[[B]]\n[[C]]", [("A", "", ""), ("B", "", ""), ("C", "", "")]), ("[[a#frag]] [[b]] [[c|alias]]", [("a", "frag", ""), ("b", "", ""), ("c", "", "alias")]), # Fragment capture is `[^\]|]+` (no # exclusion) so the second # is part of the fragment ("[[note#h1#h2]]", [("note", "h1#h2", "")]), ("[[a||b]]", [("a", "", "|b")]), ("[[a]]\\n[[b]]", [("a", "", ""), ("b", "", "")]), ("[[a]]\t[[b]]", [("a", "", ""), ("b", "", "")]), ("preface [[only]]", [("only", "", "")]), ("[[only]] postscript", [("only", "", "")]), # Target capture `[^\]|#]+` does not exclude newlines in Python re by default ("[[multi\nline]]", [("multi\nline", "", "")]), ("[[a]] [[b#c|d]] [[e|f]]", [("a", "", ""), ("b", "c", "d"), ("e", "", "f")]), ("[[has.periods.in.name]]", [("has.periods.in.name", "", "")]), ("[[trailing/slash/]]", [("trailing/slash/", "", "")]), ("[[/leading/slash]]", [("/leading/slash", "", "")]), ("[[a&b]]", [("a&b", "", "")]), ("[[a%20b]]", [("a%20b", "", "")]), ("[[a(b)c]]", [("a(b)c", "", "")]), ("[[a[b]c]]", []), # [ in target ends match ] class TestExtractWikilinks: @pytest.mark.parametrize("body, expected", _WIKILINK_FIXTURES) def test_wikilink_extraction( self, body: str, expected: list[tuple[str, str, str]] ) -> None: assert extract_wikilinks(body) == expected def test_returns_list(self) -> None: assert isinstance(extract_wikilinks(""), list) def test_count_at_least_50_fixtures(self) -> None: assert len(_WIKILINK_FIXTURES) >= 50 # ============================================================================ # TestExtractMdLinks # ============================================================================ class TestExtractMdLinks: def test_simple_md_link(self) -> None: assert extract_md_links("[text](file.md)") == [("file.md", "", "text")] def test_relative_path(self) -> None: assert extract_md_links("[text](../foo/bar.md)") == [("../foo/bar.md", "", "text")] def test_dot_slash_prefix(self) -> None: assert extract_md_links("[text](./foo.md)") == [("./foo.md", "", "text")] def test_with_fragment(self) -> None: assert extract_md_links("[text](foo.md#heading)") == [("foo.md", "heading", "text")] def test_http_link_ignored(self) -> None: assert extract_md_links("[text](https://example.com/foo.md)") == [] def test_mailto_ignored(self) -> None: assert extract_md_links("[text](mailto:foo@example.com)") == [] def test_non_md_extension_ignored(self) -> None: assert extract_md_links("[text](file.txt)") == [] def test_multiple_links(self) -> None: out = extract_md_links("[a](a.md) and [b](b.md)") assert out == [("a.md", "", "a"), ("b.md", "", "b")] def test_no_links(self) -> None: assert extract_md_links("no links here") == [] def test_empty_string(self) -> None: assert extract_md_links("") == [] def test_returns_list(self) -> None: assert isinstance(extract_md_links(""), list) # ============================================================================ # TestNormaliseRelPath (path resolution safety) # ============================================================================ class TestNormaliseRelPath: def test_simple_relative(self) -> None: assert _normalise_rel_path("inbox", "note.md") == "inbox/note.md" def test_parent_dir(self) -> None: assert _normalise_rel_path("inbox/captures", "../note.md") == "inbox/note.md" def test_dot_slash(self) -> None: assert _normalise_rel_path("inbox", "./note.md") == "inbox/note.md" def test_absolute_treated_as_vault_root(self) -> None: assert _normalise_rel_path("inbox", "/projects/note.md") == "projects/note.md" def test_escape_attempt_rejected(self) -> None: # Trying to escape vault root with too many ../ assert _normalise_rel_path("inbox", "../../../etc/passwd.md") is None def test_escape_from_root_rejected(self) -> None: assert _normalise_rel_path("", "../foo.md") is None def test_empty_source_dir(self) -> None: assert _normalise_rel_path("", "note.md") == "note.md" def test_nested_relative(self) -> None: assert ( _normalise_rel_path("projects/born-free", "../../inbox/x.md") == "inbox/x.md" ) # ============================================================================ # TestStripFrontmatter # ============================================================================ class TestStripFrontmatter: def test_strips_frontmatter(self) -> None: text = "---\ntitle: x\n---\nBody text." assert _strip_frontmatter(text) == "\nBody text." def test_no_frontmatter_returned_unchanged(self) -> None: text = "# Heading\n\nBody." assert _strip_frontmatter(text) == text def test_empty_string(self) -> None: assert _strip_frontmatter("") == "" def test_only_frontmatter(self) -> None: text = "---\ntitle: x\n---\n" assert _strip_frontmatter(text) == "\n" def test_dots_terminator(self) -> None: text = "---\ntitle: x\n...\nBody." assert _strip_frontmatter(text) == "\nBody." def test_unterminated_frontmatter_returned_unchanged(self) -> None: text = "---\ntitle: x\nNo terminator." assert _strip_frontmatter(text) == text # ============================================================================ # TestBuildLinkIndex (integration) # ============================================================================ class TestBuildLinkIndex: def test_empty_vault(self, tmp_path: pathlib.Path) -> None: idx = build_link_index(tmp_path) assert idx.notes_indexed == 0 assert idx.forward == {} assert idx.backward == {} def test_single_note_no_links(self, tmp_path: pathlib.Path) -> None: _make_vault(tmp_path, {"a.md": b"# A\n\nNo links."}) idx = build_link_index(tmp_path) assert idx.notes_indexed == 1 assert idx.forward["a.md"] == [] def test_wikilink_resolves(self, tmp_path: pathlib.Path) -> None: _make_vault(tmp_path, { "a.md": b"# A\n\nSee [[B]].", "b.md": b"# B", }) idx = build_link_index(tmp_path) links = idx.outgoing("a.md") assert len(links) == 1 assert links[0].kind == "wikilink" assert links[0].target == "b.md" assert links[0].broken is False def test_case_insensitive_wikilink(self, tmp_path: pathlib.Path) -> None: _make_vault(tmp_path, { "a.md": b"[[NOTE]]", "Note.md": b"# Note", }) idx = build_link_index(tmp_path) links = idx.outgoing("a.md") assert len(links) == 1 assert links[0].target == "Note.md" def test_broken_wikilink(self, tmp_path: pathlib.Path) -> None: _make_vault(tmp_path, {"a.md": b"[[NonExistent]]"}) idx = build_link_index(tmp_path) links = idx.outgoing("a.md") assert len(links) == 1 assert links[0].broken is True assert links[0].target is None assert links[0] in idx.broken_links def test_md_link_resolves(self, tmp_path: pathlib.Path) -> None: _make_vault(tmp_path, { "inbox/a.md": b"See [B](../projects/b.md).", "projects/b.md": b"# B", }) idx = build_link_index(tmp_path) links = idx.outgoing("inbox/a.md") assert len(links) == 1 assert links[0].kind == "md_link" assert links[0].target == "projects/b.md" def test_md_link_with_fragment(self, tmp_path: pathlib.Path) -> None: _make_vault(tmp_path, { "a.md": b"[B](b.md#heading)", "b.md": b"# B", }) idx = build_link_index(tmp_path) links = idx.outgoing("a.md") assert links[0].fragment == "heading" def test_md_link_escape_attempt_rejected(self, tmp_path: pathlib.Path) -> None: _make_vault(tmp_path, { "inbox/a.md": b"[escape](../../../etc/passwd.md)", }) idx = build_link_index(tmp_path) # Cannot resolve outside vault → broken links = idx.outgoing("inbox/a.md") assert len(links) == 1 assert links[0].broken is True def test_backward_index(self, tmp_path: pathlib.Path) -> None: _make_vault(tmp_path, { "a.md": b"[[Target]]", "b.md": b"[[Target]]", "Target.md": b"# Target", }) idx = build_link_index(tmp_path) incoming = idx.incoming("Target.md") sources = {link.source for link in incoming} assert sources == {"a.md", "b.md"} def test_entity_index(self, tmp_path: pathlib.Path) -> None: _make_vault(tmp_path, { "a.md": _note(fm="entity:\n - Alice\n - Bob", body="A"), "b.md": _note(fm="entity:\n - Bob", body="B"), "c.md": _note(fm="entity:\n - Charlie", body="C"), }) idx = build_link_index(tmp_path) bob_notes = idx.entity_index["Bob"] assert bob_notes == {"a.md", "b.md"} assert idx.co_entity("a.md") == {"b.md"} assert idx.co_entity("c.md") == set() def test_duplicate_basenames_recorded(self, tmp_path: pathlib.Path) -> None: _make_vault(tmp_path, { "folder1/Note.md": b"# Note 1", "folder2/Note.md": b"# Note 2", }) idx = build_link_index(tmp_path) assert "note" in idx.duplicate_basenames assert len(idx.duplicate_basenames["note"]) == 2 def test_frontmatter_links_not_extracted(self, tmp_path: pathlib.Path) -> None: """Wikilinks inside YAML frontmatter must NOT be treated as body links.""" _make_vault(tmp_path, { "a.md": _note(fm="title: '[[FakeWikilink]]'", body="Real [[B]]"), "b.md": b"# B", }) idx = build_link_index(tmp_path) links = idx.outgoing("a.md") # Only the body wikilink should be extracted assert len(links) == 1 assert links[0].raw == "B" def test_ignored_directories_pruned(self, tmp_path: pathlib.Path) -> None: _make_vault(tmp_path, { "templates/README.md": b"[[Should Not Be Indexed]]", "real.md": b"# Real", }) idx = build_link_index(tmp_path) assert idx.notes_indexed == 1 assert "templates/README.md" not in idx.forward def test_hidden_directories_pruned(self, tmp_path: pathlib.Path) -> None: _make_vault(tmp_path, { ".muse/internal.md": b"# Internal", ".obsidian/notes.md": b"# Obsidian", "real.md": b"# Real", }) idx = build_link_index(tmp_path) assert idx.notes_indexed == 1 def test_nonexistent_root_raises(self, tmp_path: pathlib.Path) -> None: with pytest.raises(ValueError): build_link_index(tmp_path / "does-not-exist") def test_file_as_root_raises(self, tmp_path: pathlib.Path) -> None: f = tmp_path / "file.md" f.write_bytes(b"") with pytest.raises(ValueError): build_link_index(f) # ============================================================================ # TestResolvedLinkInvariants # ============================================================================ class TestResolvedLinkInvariants: def test_resolved_link_is_frozen(self) -> None: import dataclasses link = ResolvedLink(source="a.md", target="b.md", raw="b", kind="wikilink") with pytest.raises((dataclasses.FrozenInstanceError, AttributeError, TypeError)): link.target = "hacked.md" # type: ignore[misc] def test_broken_link_has_no_target(self) -> None: link = ResolvedLink( source="a.md", target=None, raw="missing", kind="wikilink", broken=True ) assert link.broken is True assert link.target is None # ============================================================================ # TestPerformance # ============================================================================ class TestPerformance: def test_build_under_10s_on_500_notes(self, tmp_path: pathlib.Path) -> None: """Performance test scaled down to 500 notes for CI speed; real bar is 5k in <10s, so 500 should be ~1s. Skip if system is heavily loaded.""" files: dict[str, bytes | str] = {} for i in range(500): files[f"note_{i:04d}.md"] = ( f"# Note {i}\n\nSee [[note_{(i + 1) % 500:04d}]] " f"and [neighbour](./note_{(i + 2) % 500:04d}.md)." ).encode() _make_vault(tmp_path, files) start = time.monotonic() idx = build_link_index(tmp_path) elapsed = time.monotonic() - start assert idx.notes_indexed == 500 assert elapsed < 5.0, f"500 notes took {elapsed:.2f}s (budget 5s)" # ============================================================================ # TestStress # ============================================================================ class TestStress: def test_large_note_indexed(self, tmp_path: pathlib.Path) -> None: body = "[[target]]\n" * 5000 _make_vault(tmp_path, { "big.md": body.encode(), "target.md": b"# T", }) idx = build_link_index(tmp_path) assert len(idx.outgoing("big.md")) == 5000 assert idx.outgoing("big.md")[0].broken is False def test_oversized_note_skipped( self, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch ) -> None: # Lower the cap so we can test without writing 16 MiB. from muse.plugins.knowtation import link_index as li monkeypatch.setattr(li, "_MAX_NOTE_BYTES", 100) _make_vault(tmp_path, { "huge.md": b"[[target]]\n" * 100, # >100 bytes "target.md": b"# T", }) idx = build_link_index(tmp_path) # huge.md was counted (notes_indexed = 2) but its links are not parsed assert idx.outgoing("huge.md") == [] # ============================================================================ # TestSecurity # ============================================================================ class TestSecurity: def test_path_traversal_in_md_link_rejected(self, tmp_path: pathlib.Path) -> None: _make_vault(tmp_path, { "a.md": b"[escape](../../../etc/passwd.md)", }) idx = build_link_index(tmp_path) assert idx.outgoing("a.md")[0].broken is True def test_absolute_path_with_etc_resolves_inside_vault_only( self, tmp_path: pathlib.Path ) -> None: _make_vault(tmp_path, { "a.md": b"[etc](/etc/passwd.md)", }) idx = build_link_index(tmp_path) # Resolves to vault-root "etc/passwd.md" which doesn't exist → broken assert idx.outgoing("a.md")[0].broken is True def test_binary_content_does_not_crash(self, tmp_path: pathlib.Path) -> None: _make_vault(tmp_path, { "binary.md": bytes(range(256)) * 10, "real.md": b"# Real", }) try: idx = build_link_index(tmp_path) assert idx.notes_indexed == 2 except Exception as exc: pytest.fail(f"Binary content crashed indexer: {exc}") def test_nul_bytes_in_body(self, tmp_path: pathlib.Path) -> None: _make_vault(tmp_path, { "nul.md": b"[[target]]\x00\x00\x00", "target.md": b"# T", }) try: idx = build_link_index(tmp_path) assert idx.notes_indexed == 2 except Exception as exc: pytest.fail(f"NUL bytes crashed indexer: {exc}") def test_yaml_injection_in_entity_field(self, tmp_path: pathlib.Path) -> None: malicious_fm = "entity: !!python/object/apply:os.system ['echo PWNED']" _make_vault(tmp_path, { "a.md": _note(fm=malicious_fm, body="[[x]]"), "x.md": b"# X", }) try: idx = build_link_index(tmp_path) assert idx.notes_indexed == 2 except Exception: pass # Acceptable to raise; must NOT execute the payload. def test_deeply_nested_directories(self, tmp_path: pathlib.Path) -> None: deep = "/".join([f"d{i}" for i in range(20)]) _make_vault(tmp_path, { f"{deep}/note.md": b"[[other]]", "other.md": b"# Other", }) idx = build_link_index(tmp_path) assert idx.notes_indexed == 2 # ============================================================================ # TestDataIntegrityRealVault # ============================================================================ _REAL_VAULTS: tuple[pathlib.Path, ...] = ( pathlib.Path("/Users/aaronrenecarvajal/knowtation/vault"), pathlib.Path("/Users/aaronrenecarvajal/knowtation-vault"), ) def _find_real_vault() -> pathlib.Path | None: for v in _REAL_VAULTS: if v.exists() and v.is_dir(): return v return None class TestDataIntegrityRealVault: """Round-trip on a real Knowtation vault when available.""" @pytest.mark.skipif(_find_real_vault() is None, reason="No real vault available") def test_real_vault_indexes_without_error(self) -> None: vault = _find_real_vault() assert vault is not None idx = build_link_index(vault) assert idx.notes_indexed > 0 @pytest.mark.skipif(_find_real_vault() is None, reason="No real vault available") def test_real_vault_completes_under_10s(self) -> None: vault = _find_real_vault() assert vault is not None start = time.monotonic() build_link_index(vault) elapsed = time.monotonic() - start assert elapsed < 10.0, f"Real vault took {elapsed:.2f}s (budget 10s)"