aaronrene / knowtation public
markdown.mjs
112 lines 3.6 KB
Raw
sha256:65ccb454656ea5acdea0a10e559b78bcde1eb6ff753ecc2911bc99d1c3d7cadd feat(calendar): enforce agent context tiers in retrieval AP… Human minor ⚠ breaking 1 day ago
1 /**
2 * Generic Markdown importer. Path to file or folder; add source: markdown, date if missing.
3 */
4
5 import fs from 'fs';
6 import path from 'path';
7 import { writeNote } from '../write.mjs';
8 import { parseFrontmatterAndBody, normalizeSlug } from '../vault.mjs';
9
10 function isMarkdownFilename(nameOrPath) {
11 const base = path.basename(nameOrPath).toLowerCase();
12 return base.endsWith('.md') || base.endsWith('.markdown');
13 }
14
15 /**
16 * @param {string} input - Path to .md file or folder of .md files
17 * @param {{
18 * vaultPath: string,
19 * outputBase: string,
20 * project?: string,
21 * tags: string[],
22 * dryRun: boolean,
23 * onProgress?: (p: { progress: number, total?: number, message?: string }) => void | Promise<void>
24 * }} ctx
25 * @returns {Promise<{ imported: { path: string, source_id?: string }[], count: number }>}
26 */
27 export async function importMarkdown(input, ctx) {
28 const { vaultPath, outputBase, project, tags, dryRun, onProgress } = ctx;
29 const absInput = path.isAbsolute(input) ? input : path.resolve(process.cwd(), input);
30 if (!fs.existsSync(absInput)) {
31 throw new Error(`Input not found: ${input}`);
32 }
33
34 const files = [];
35 if (fs.statSync(absInput).isFile()) {
36 if (isMarkdownFilename(absInput)) files.push({ fullPath: absInput, relPath: path.basename(absInput) });
37 } else {
38 walkMarkdown(absInput, absInput, '', files);
39 }
40
41 const imported = [];
42 const now = new Date().toISOString().slice(0, 10);
43 const totalFiles = files.length;
44 let lastProgressIdx = 0;
45 let lastProgressMs = 0;
46
47 for (let fi = 0; fi < files.length; fi++) {
48 const { fullPath, relPath } = files[fi];
49 const outputRel = path.join(outputBase, relPath).replace(/\\/g, '/');
50
51 const content = fs.readFileSync(fullPath, 'utf8');
52 const { frontmatter, body } = parseFrontmatterAndBody(content);
53 const dateRaw = frontmatter.date || frontmatter.created || now;
54 const date = normalizeDate(dateRaw) || now;
55 const merged = {
56 ...frontmatter,
57 source: 'markdown',
58 date,
59 ...(project && { project: normalizeSlug(project) }),
60 ...(tags.length && { tags }),
61 };
62 if (typeof merged.tags === 'string') merged.tags = tags;
63 else if (Array.isArray(merged.tags)) merged.tags = [...new Set([...merged.tags, ...tags])];
64 else merged.tags = tags;
65
66 if (!dryRun) {
67 writeNote(vaultPath, outputRel, {
68 body,
69 frontmatter: Object.fromEntries(
70 Object.entries(merged).filter(([, v]) => v !== undefined && v !== null && v !== '')
71 ),
72 });
73 }
74 imported.push({ path: outputRel, source_id: relPath });
75
76 if (onProgress && totalFiles > 0) {
77 const n = fi + 1;
78 const nowMs = Date.now();
79 const force = n === 1 || n === totalFiles || totalFiles <= 10;
80 if (force || n - lastProgressIdx >= 10 || nowMs - lastProgressMs >= 5000) {
81 lastProgressIdx = n;
82 lastProgressMs = nowMs;
83 await onProgress({
84 progress: n,
85 total: totalFiles,
86 message: `markdown import ${n}/${totalFiles}: ${relPath}`,
87 });
88 }
89 }
90 }
91
92 return { imported, count: imported.length };
93 }
94
95 function normalizeDate(v) {
96 if (!v) return null;
97 const d = new Date(v);
98 if (isNaN(d.getTime())) return null;
99 return d.toISOString().slice(0, 10);
100 }
101
102 function walkMarkdown(rootDir, dir, relDir, out) {
103 const entries = fs.readdirSync(dir, { withFileTypes: true });
104 for (const e of entries) {
105 const rel = relDir ? `${relDir}/${e.name}` : e.name;
106 if (e.isDirectory()) {
107 walkMarkdown(rootDir, path.join(dir, e.name), rel, out);
108 } else if (isMarkdownFilename(e.name)) {
109 out.push({ fullPath: path.join(dir, e.name), relPath: rel.replace(/\\/g, '/') });
110 }
111 }
112 }
File History 2 commits
sha256:65ccb454656ea5acdea0a10e559b78bcde1eb6ff753ecc2911bc99d1c3d7cadd feat(calendar): enforce agent context tiers in retrieval AP… Human minor ⚠ 1 day ago
sha256:9103f98c89257ed2b01c237cea895dabb3e85ea337dccb1161c175e4422355b6 docs: accept Calendar Events v0 spec with Phase 0 security … Human 1 day ago