aaronrene / knowtation public
retrieval-cost-demo.mjs
94 lines 3.7 KB
Raw
sha256:65ccb454656ea5acdea0a10e559b78bcde1eb6ff753ecc2911bc99d1c3d7cadd feat(calendar): enforce agent context tiers in retrieval AP… Human minor ⚠ breaking 1 day ago
1 #!/usr/bin/env node
2 /**
3 * Retrieval cost demo: compare "standard" (broad fetch) vs "refined" (tiered) retrieval.
4 * Measures output size (chars) and estimated tokens so we can show cost savings in docs.
5 *
6 * Usage (from repo root):
7 * node scripts/retrieval-cost-demo.mjs [query]
8 * Query defaults to "project" if omitted.
9 *
10 * Uses lib directly (no CLI subprocess). Set KNOWTATION_VAULT_PATH or ensure config/local.yaml has vault_path.
11 */
12
13 import path from 'path';
14 import { fileURLToPath } from 'url';
15
16 const __dirname = path.dirname(fileURLToPath(import.meta.url));
17 const ROOT = path.join(__dirname, '..');
18 if (!process.env.KNOWTATION_VAULT_PATH) {
19 process.env.KNOWTATION_VAULT_PATH = path.join(ROOT, 'vault');
20 }
21
22 import { loadConfig } from '../lib/config.mjs';
23 import { runListNotes } from '../lib/list-notes.mjs';
24 import { readNote } from '../lib/vault.mjs';
25
26 function countChars(str) {
27 return typeof str === 'string' ? str.length : 0;
28 }
29
30 /** Rough token estimate: ~4 chars per token for English/markdown. */
31 function estTokens(chars) {
32 return Math.round(chars / 4);
33 }
34
35 function main() {
36 const query = process.argv[2]?.trim() || 'project';
37 console.log('Retrieval cost demo: Standard vs Refined');
38 console.log('Query:', JSON.stringify(query));
39 console.log('');
40
41 let config;
42 try {
43 config = loadConfig(ROOT);
44 } catch (e) {
45 console.error('Config error:', e.message);
46 console.error('Set KNOWTATION_VAULT_PATH or create config/local.yaml with vault_path.');
47 process.exit(2);
48 }
49
50 let standardChars = 0;
51 let refinedChars = 0;
52
53 // Standard: list-notes with full metadata (path+metadata), limit 10, then get full note for up to 5
54 const listStandard = runListNotes(config, { limit: 10, fields: 'path+metadata' });
55 const standardJson = JSON.stringify(listStandard);
56 standardChars += countChars(standardJson);
57 const standardPaths = (listStandard.notes || []).map((n) => n.path).slice(0, 5);
58
59 for (const p of standardPaths) {
60 try {
61 const note = readNote(config.vault_path, p);
62 standardChars += countChars(JSON.stringify({ path: note.path, frontmatter: note.frontmatter, body: note.body }));
63 } catch (_) {}
64 }
65
66 // Refined: list-notes with path only, limit 3, then get full note for first path only
67 const listRefined = runListNotes(config, { limit: 3, fields: 'path' });
68 const refinedJson = JSON.stringify(listRefined);
69 refinedChars += countChars(refinedJson);
70 const refinedPath = (listRefined.notes || [])[0]?.path;
71
72 if (refinedPath) {
73 try {
74 const note = readNote(config.vault_path, refinedPath);
75 refinedChars += countChars(JSON.stringify({ path: note.path, frontmatter: note.frontmatter, body: note.body }));
76 } catch (_) {}
77 }
78
79 const stdTokens = estTokens(standardChars);
80 const refTokens = estTokens(refinedChars);
81 const savings = standardChars > 0 ? Math.round((1 - refinedChars / standardChars) * 100) : 0;
82
83 console.log('Strategy | Chars | Est. tokens (÷4) | Est. cost @ $0.50/1M');
84 console.log('----------------------|--------|------------------|----------------------');
85 console.log(`Standard (10+metadata, ${standardPaths.length} get-note) | ${String(standardChars).padStart(6)} | ${String(stdTokens).padStart(16)} | $${(stdTokens / 1e6 * 0.5).toFixed(4)}`);
86 console.log(`Refined (3 path, 1 get-note) | ${String(refinedChars).padStart(6)} | ${String(refTokens).padStart(16)} | $${(refTokens / 1e6 * 0.5).toFixed(4)}`);
87 console.log('');
88 console.log(`Token reduction: ${savings}% (refined vs standard).`);
89 console.log('');
90 console.log('Standard = list-notes --limit 10 (path+metadata) + get-note for up to 5 paths.');
91 console.log('Refined = list-notes --limit 3 --fields path + get-note for 1 path.');
92 }
93
94 main();
File History 2 commits
sha256:65ccb454656ea5acdea0a10e559b78bcde1eb6ff753ecc2911bc99d1c3d7cadd feat(calendar): enforce agent context tiers in retrieval AP… Human minor ⚠ 1 day ago
sha256:9103f98c89257ed2b01c237cea895dabb3e85ea337dccb1161c175e4422355b6 docs: accept Calendar Events v0 spec with Phase 0 security … Human 2 days ago