aaronrene / knowtation public
config.mjs
388 lines 16.9 KB
Raw
sha256:65ccb454656ea5acdea0a10e559b78bcde1eb6ff753ecc2911bc99d1c3d7cadd feat(calendar): enforce agent context tiers in retrieval AP… Human minor ⚠ breaking 1 day ago
1 /**
2 * Config loader: config/local.yaml + env overrides. SPEC §4.4.
3 * Env overrides apply after file. vault_path is required.
4 * Multi-vault (Phase 15): when hub_vaults.yaml is absent, single vault "default" from vault_path.
5 */
6
7 import fs from 'fs';
8 import path from 'path';
9 import yaml from 'js-yaml';
10 import { readHubVaults } from './hub-vaults.mjs';
11 import { getRepoRoot } from './repo-root.mjs';
12
13 const projectRoot = getRepoRoot();
14
15 const ENV_VAULT = 'KNOWTATION_VAULT_PATH';
16 const ENV_QDRANT = 'QDRANT_URL';
17 const ENV_DATA_DIR = 'KNOWTATION_DATA_DIR';
18 const ENV_VECTOR_STORE = 'KNOWTATION_VECTOR_STORE';
19 const ENV_MEMORY_URL = 'KNOWTATION_MEMORY_URL';
20 const ENV_MEMORY_ENABLED = 'KNOWTATION_MEMORY_ENABLED';
21 const ENV_MEMORY_PROVIDER = 'KNOWTATION_MEMORY_PROVIDER';
22 const ENV_AIR_ENDPOINT = 'KNOWTATION_AIR_ENDPOINT';
23 const ENV_OLLAMA_URL = 'OLLAMA_URL';
24 const ENV_EMBEDDING_PROVIDER = 'EMBEDDING_PROVIDER';
25 const ENV_EMBEDDING_MODEL = 'EMBEDDING_MODEL';
26 const ENV_TRANSCODE_OVERSIZED = 'KNOWTATION_TRANSCODE_OVERSIZED';
27 const ENV_MUSE_URL = 'MUSE_URL';
28
29 /** Default embed model name when YAML/env omits `model` (must match provider). */
30 function defaultEmbeddingModelForProvider(provider) {
31 const p = String(provider || 'ollama').toLowerCase();
32 if (p === 'openai') return 'text-embedding-3-small';
33 if (p === 'voyage') return 'voyage-4-lite';
34 return 'nomic-embed-text';
35 }
36
37 const DEFAULT_IGNORE = ['templates', 'meta', 'node_modules', '.git'];
38
39 /**
40 * Read transcription.* from config/local.yaml only (no vault_path required).
41 * Used by lib/transcribe.mjs on hosted bridge where full loadConfig may be skipped.
42 * @param {string} [cwd]
43 * @returns {{ provider: string, model: string, transcode_oversized: boolean }}
44 */
45 export function readTranscriptionYaml(cwd = projectRoot) {
46 const configPath = path.join(cwd, 'config', 'local.yaml');
47 let t = {};
48 if (fs.existsSync(configPath)) {
49 try {
50 const raw = yaml.load(fs.readFileSync(configPath, 'utf8')) || {};
51 if (raw.transcription && typeof raw.transcription === 'object') {
52 t = raw.transcription;
53 }
54 } catch (_) {
55 /* ignore invalid yaml for this optional slice */
56 }
57 }
58 return {
59 provider: t.provider || 'openai',
60 model: t.model || 'whisper-1',
61 transcode_oversized: t.transcode_oversized !== false,
62 };
63 }
64
65 /**
66 * Load config from config/local.yaml (if present) then apply env overrides.
67 *
68 * **AIR config** (`air` key in local.yaml):
69 * ```yaml
70 * air:
71 * enabled: true # master switch; default false
72 * required: true # hard-fail: throw AttestationRequiredError when endpoint fails; default false
73 * endpoint: https://... # attestation endpoint URL; falls back to KNOWTATION_AIR_ENDPOINT env var
74 * ```
75 * When `air.required=true` a write or export is rejected if the attestation endpoint is
76 * unreachable or returns a non-OK response. Default (`false`) is non-blocking: a placeholder
77 * id is logged and the operation proceeds (backward-compatible).
78 *
79 * @param {string} [cwd] - Working directory (default: project root)
80 * @returns {{ vault_path: string, qdrant_url?: string, vector_store?: string, data_dir: string, embedding?: object, memory?: object, air?: { enabled: boolean, required: boolean, endpoint: string|undefined }, ignore?: string[] }} embedding.ollama_url from YAML or OLLAMA_URL env when set
81 * @throws if vault_path is missing after load
82 */
83 export function loadConfig(cwd = projectRoot) {
84 const configPath = path.join(cwd, 'config', 'local.yaml');
85 let config = {};
86
87 if (fs.existsSync(configPath)) {
88 try {
89 const raw = fs.readFileSync(configPath, 'utf8');
90 config = yaml.load(raw) || {};
91 } catch (e) {
92 throw new Error(`Invalid config at ${configPath}: ${e.message}`);
93 }
94 }
95
96 // Env overrides (SPEC: env overrides, then config)
97 if (process.env[ENV_VAULT]) config.vault_path = process.env[ENV_VAULT];
98 if (process.env[ENV_QDRANT]) config.qdrant_url = process.env[ENV_QDRANT];
99 if (process.env[ENV_DATA_DIR]) config.data_dir = process.env[ENV_DATA_DIR];
100 if (process.env[ENV_VECTOR_STORE]) config.vector_store = process.env[ENV_VECTOR_STORE];
101 if (process.env[ENV_MEMORY_URL]) config.memory = { ...(config.memory || {}), url: process.env[ENV_MEMORY_URL] };
102 if (process.env[ENV_MEMORY_ENABLED] === 'true') config.memory = { ...(config.memory || {}), enabled: true };
103 if (process.env[ENV_MEMORY_ENABLED] === 'false') config.memory = { ...(config.memory || {}), enabled: false };
104 if (process.env[ENV_MEMORY_PROVIDER]) config.memory = { ...(config.memory || {}), provider: process.env[ENV_MEMORY_PROVIDER] };
105 if (process.env[ENV_AIR_ENDPOINT]) config.air = { ...config.air, endpoint: process.env[ENV_AIR_ENDPOINT] };
106 if (process.env[ENV_TRANSCODE_OVERSIZED] === '0' || process.env[ENV_TRANSCODE_OVERSIZED] === 'false') {
107 config.transcription = { ...(config.transcription || {}), transcode_oversized: false };
108 }
109 if (process.env[ENV_TRANSCODE_OVERSIZED] === '1' || process.env[ENV_TRANSCODE_OVERSIZED] === 'true') {
110 config.transcription = { ...(config.transcription || {}), transcode_oversized: true };
111 }
112
113 // Hub Setup overrides (optional): data_dir/hub_setup.yaml can set vault_path and vault.git
114 const dataDirPath = path.resolve(cwd, config.data_dir || 'data');
115 const hubSetupPath = path.join(dataDirPath, 'hub_setup.yaml');
116 if (fs.existsSync(hubSetupPath)) {
117 try {
118 const setupRaw = fs.readFileSync(hubSetupPath, 'utf8');
119 const setup = yaml.load(setupRaw) || {};
120 // Hub writes vault_path here; operator/tests use KNOWTATION_VAULT_PATH — that must win (SPEC: env overrides).
121 if (setup.vault_path != null && !process.env[ENV_VAULT]) {
122 config.vault_path = setup.vault_path;
123 }
124 if (setup.vault?.git && typeof setup.vault.git === 'object') {
125 config.vault = config.vault || {};
126 config.vault.git = { ...(config.vault.git || {}), ...setup.vault.git };
127 }
128 } catch (_) { /* ignore invalid hub_setup */ }
129 }
130
131 /** Muse thin bridge: optional `muse.url` in local.yaml; `MUSE_URL` env wins when set. */
132 const museRaw = config.muse && typeof config.muse === 'object' ? config.muse : {};
133 let museUrlMerged =
134 typeof museRaw.url === 'string' ? museRaw.url.trim().replace(/\/+$/, '') : '';
135 if (process.env[ENV_MUSE_URL] != null && String(process.env[ENV_MUSE_URL]).trim() !== '') {
136 museUrlMerged = String(process.env[ENV_MUSE_URL]).trim().replace(/\/+$/, '');
137 }
138 config.muse = museUrlMerged ? { url: museUrlMerged } : {};
139
140 const vaultPath = config.vault_path;
141 if (!vaultPath || typeof vaultPath !== 'string') {
142 throw new Error('vault_path is required. Set in config/local.yaml or env KNOWTATION_VAULT_PATH.');
143 }
144
145 const resolvedVault = path.isAbsolute(vaultPath) ? vaultPath : path.resolve(cwd, vaultPath);
146 if (!fs.existsSync(resolvedVault) || !fs.statSync(resolvedVault).isDirectory()) {
147 throw new Error(`Vault path does not exist or is not a directory: ${resolvedVault}`);
148 }
149
150 let vaultList = readHubVaults(dataDirPath, cwd);
151 if (vaultList.length === 0) {
152 vaultList = [{ id: 'default', path: resolvedVault, label: undefined }];
153 }
154
155 /**
156 * Resolve vault id to absolute path. Returns undefined if vault id not in list.
157 * @param {string} vaultId
158 * @returns {string | undefined}
159 */
160 function resolveVaultPath(vaultId) {
161 const v = vaultList.find((e) => e.id === vaultId);
162 return v ? v.path : undefined;
163 }
164
165 const embeddingYaml = config.embedding && typeof config.embedding === 'object' ? config.embedding : null;
166 let embeddingProvider = embeddingYaml?.provider || 'ollama';
167 if (process.env[ENV_EMBEDDING_PROVIDER] != null && String(process.env[ENV_EMBEDDING_PROVIDER]).trim() !== '') {
168 embeddingProvider = String(process.env[ENV_EMBEDDING_PROVIDER]).trim().toLowerCase();
169 }
170 let embeddingModel = embeddingYaml?.model;
171 if (process.env[ENV_EMBEDDING_MODEL] != null && String(process.env[ENV_EMBEDDING_MODEL]).trim() !== '') {
172 embeddingModel = String(process.env[ENV_EMBEDDING_MODEL]).trim();
173 }
174 if (embeddingModel == null || String(embeddingModel).trim() === '') {
175 embeddingModel = defaultEmbeddingModelForProvider(embeddingProvider);
176 }
177 const embedding = {
178 provider: embeddingProvider,
179 model: embeddingModel,
180 ollama_url: embeddingYaml?.ollama_url,
181 };
182 if (process.env[ENV_OLLAMA_URL]) {
183 embedding.ollama_url = process.env[ENV_OLLAMA_URL];
184 }
185
186 return {
187 vault_path: resolvedVault,
188 vaultList,
189 resolveVaultPath,
190 qdrant_url: config.qdrant_url,
191 vector_store: config.vector_store || 'qdrant',
192 data_dir: path.resolve(cwd, config.data_dir || 'data'),
193 embedding,
194 indexer: config.indexer && typeof config.indexer === 'object'
195 ? {
196 chunk_size: config.indexer.chunk_size ?? 2048,
197 chunk_overlap: config.indexer.chunk_overlap ?? 256,
198 }
199 : { chunk_size: 2048, chunk_overlap: 256 },
200 transcription: config.transcription && typeof config.transcription === 'object'
201 ? {
202 provider: config.transcription.provider || 'openai',
203 model: config.transcription.model || 'whisper-1',
204 transcode_oversized: config.transcription.transcode_oversized !== false,
205 }
206 : { provider: 'openai', model: 'whisper-1', transcode_oversized: true },
207 memory: config.memory && typeof config.memory === 'object'
208 ? {
209 enabled: config.memory.enabled === true,
210 provider: config.memory.provider || 'file',
211 url: config.memory.url || process.env.KNOWTATION_MEMORY_URL,
212 retention_days: config.memory.retention_days ?? null,
213 capture: Array.isArray(config.memory.capture) ? config.memory.capture : undefined,
214 scope: config.memory.scope === 'global' ? 'global' : 'vault',
215 encrypt: config.memory.encrypt === true,
216 secret: config.memory.secret || undefined,
217 supabase_url: config.memory.supabase_url || process.env.KNOWTATION_SUPABASE_URL || undefined,
218 supabase_key: config.memory.supabase_key || process.env.KNOWTATION_SUPABASE_KEY || undefined,
219 }
220 : { enabled: false, provider: 'file', url: undefined, retention_days: null, capture: undefined, scope: 'vault', encrypt: false, secret: undefined, supabase_url: undefined, supabase_key: undefined },
221 air: config.air && typeof config.air === 'object'
222 ? {
223 enabled: config.air.enabled === true,
224 required: config.air.required === true,
225 endpoint: config.air.endpoint || process.env.KNOWTATION_AIR_ENDPOINT,
226 }
227 : { enabled: false, required: false, endpoint: undefined },
228 vault_git: config.vault?.git && typeof config.vault.git === 'object'
229 ? {
230 enabled: config.vault.git.enabled === true,
231 remote: config.vault.git.remote || undefined,
232 auto_commit: config.vault.git.auto_commit === true,
233 auto_push: config.vault.git.auto_push === true,
234 }
235 : { enabled: false, remote: undefined, auto_commit: false, auto_push: false },
236 mcp: (() => {
237 const mcpRaw = config.mcp && typeof config.mcp === 'object' ? config.mcp : {};
238 const envPort = process.env.KNOWTATION_MCP_HTTP_PORT;
239 const http_port =
240 envPort != null && String(envPort).trim() !== ''
241 ? parseInt(String(envPort), 10) || 3334
242 : mcpRaw.http_port ?? 3334;
243 const http_host = mcpRaw.http_host || '127.0.0.1';
244 return { http_port, http_host };
245 })(),
246 muse: config.muse && typeof config.muse === 'object' && config.muse.url
247 ? { url: String(config.muse.url).trim().replace(/\/+$/, '') }
248 : {},
249 daemon: loadDaemonConfig(config.daemon),
250 llm: loadLlmConfig(config.llm),
251 ignore: config.ignore || DEFAULT_IGNORE,
252 };
253 }
254
255 /** Chat providers selectable for the `completeChat` path (Hub Settings UI + config llm.provider). */
256 export const CHAT_PROVIDERS = Object.freeze(['deepinfra', 'openrouter', 'openai', 'anthropic', 'ollama']);
257
258 /**
259 * Validate a chat-provider value submitted from the Hub Settings UI before it is persisted.
260 *
261 * Security boundary: the persisted provider drives where note text is sent (privacy) and which
262 * account is billed. Only an empty string ("" → auto-detect) or an exact whitelisted provider is
263 * accepted; anything else is rejected so a malformed/hostile value can never be written to
264 * config/local.yaml or reach completeChat.
265 *
266 * @param {unknown} value
267 * @returns {{ ok: true, provider: string } | { ok: false, error: string }}
268 */
269 export function normalizeChatProviderInput(value) {
270 if (value == null || value === '') return { ok: true, provider: '' };
271 if (typeof value !== 'string') {
272 return { ok: false, error: 'provider must be a string' };
273 }
274 const v = value.trim().toLowerCase();
275 if (v === '') return { ok: true, provider: '' };
276 if (!CHAT_PROVIDERS.includes(v)) {
277 return {
278 ok: false,
279 error: `provider must be one of: ${CHAT_PROVIDERS.join(', ')} (or empty for auto-detect)`,
280 };
281 }
282 return { ok: true, provider: v };
283 }
284
285 /**
286 * Resolve the top-level `llm` block (the `completeChat` path) from local.yaml.
287 *
288 * Distinct from `daemon.llm` (consolidation/Discovery). `provider` selects the chat lane used by
289 * MCP summarize and Hub proposal LLM jobs; invalid values are dropped (treated as auto-detect) so a
290 * malformed config can never force an unknown provider. The env var `KNOWTATION_CHAT_PROVIDER` still
291 * takes precedence at resolution time (see lib/llm-complete.mjs).
292 *
293 * @param {object|undefined} raw — the `llm` key from the YAML file
294 * @returns {{ provider: string, openai_chat_model: string|undefined, anthropic_chat_model: string|undefined, deepinfra_chat_model: string|undefined, openrouter_chat_model: string|undefined, ollama_chat_model: string|undefined }}
295 */
296 export function loadLlmConfig(raw) {
297 const l = raw && typeof raw === 'object' ? raw : {};
298 const providerRaw = String(l.provider || '').trim().toLowerCase();
299 const provider = CHAT_PROVIDERS.includes(providerRaw) ? providerRaw : '';
300 return {
301 provider,
302 openai_chat_model: l.openai_chat_model || undefined,
303 anthropic_chat_model: l.anthropic_chat_model || undefined,
304 deepinfra_chat_model: l.deepinfra_chat_model || undefined,
305 openrouter_chat_model: l.openrouter_chat_model || undefined,
306 ollama_chat_model: l.ollama_chat_model || undefined,
307 };
308 }
309
310 const DAEMON_DEFAULTS = Object.freeze({
311 enabled: false,
312 interval_minutes: 120,
313 idle_only: true,
314 idle_threshold_minutes: 15,
315 run_on_start: false,
316 lookback_hours: 24,
317 max_events_per_pass: 200,
318 max_topics_per_pass: 10,
319 passes: { consolidate: true, verify: true, discover: false, rebuild_index: true },
320 llm: {
321 provider: null,
322 model: null,
323 api_key_env: null,
324 base_url: null,
325 max_tokens: 1024,
326 temperature: 0.2,
327 },
328 dry_run: false,
329 log_file: null,
330 max_cost_per_day_usd: null,
331 });
332
333 /**
334 * Parse daemon configuration from the raw `daemon` section of local.yaml,
335 * applying defaults and environment variable overrides.
336 *
337 * @param {object|undefined} raw — the `daemon` key from the YAML file
338 * @returns {object} fully resolved daemon config with all defaults applied
339 */
340 export function loadDaemonConfig(raw) {
341 const d = raw && typeof raw === 'object' ? raw : {};
342
343 const enabled = process.env.KNOWTATION_DAEMON_ENABLED != null
344 ? process.env.KNOWTATION_DAEMON_ENABLED === 'true'
345 : d.enabled === true;
346
347 const interval_minutes = process.env.KNOWTATION_DAEMON_INTERVAL != null
348 ? parseInt(process.env.KNOWTATION_DAEMON_INTERVAL, 10) || DAEMON_DEFAULTS.interval_minutes
349 : d.interval_minutes ?? DAEMON_DEFAULTS.interval_minutes;
350
351 const dry_run = process.env.KNOWTATION_DAEMON_DRY_RUN != null
352 ? process.env.KNOWTATION_DAEMON_DRY_RUN === 'true'
353 : d.dry_run === true;
354
355 const rawPasses = d.passes && typeof d.passes === 'object' ? d.passes : {};
356 const passes = {
357 consolidate: rawPasses.consolidate !== false,
358 verify: rawPasses.verify !== false,
359 discover: rawPasses.discover === true,
360 rebuild_index: rawPasses.rebuild_index !== false,
361 };
362
363 const rawLlm = d.llm && typeof d.llm === 'object' ? d.llm : {};
364 const llm = {
365 provider: process.env.KNOWTATION_DAEMON_LLM_PROVIDER || rawLlm.provider || null,
366 model: process.env.KNOWTATION_DAEMON_LLM_MODEL || rawLlm.model || null,
367 api_key_env: rawLlm.api_key_env || null,
368 base_url: process.env.KNOWTATION_DAEMON_LLM_BASE_URL || rawLlm.base_url || null,
369 max_tokens: rawLlm.max_tokens ?? DAEMON_DEFAULTS.llm.max_tokens,
370 temperature: rawLlm.temperature ?? DAEMON_DEFAULTS.llm.temperature,
371 };
372
373 return {
374 enabled,
375 interval_minutes,
376 idle_only: d.idle_only !== false,
377 idle_threshold_minutes: d.idle_threshold_minutes ?? DAEMON_DEFAULTS.idle_threshold_minutes,
378 run_on_start: d.run_on_start === true,
379 lookback_hours: d.lookback_hours ?? DAEMON_DEFAULTS.lookback_hours,
380 max_events_per_pass: d.max_events_per_pass ?? DAEMON_DEFAULTS.max_events_per_pass,
381 max_topics_per_pass: d.max_topics_per_pass ?? DAEMON_DEFAULTS.max_topics_per_pass,
382 passes,
383 llm,
384 dry_run,
385 log_file: d.log_file || null,
386 max_cost_per_day_usd: d.max_cost_per_day_usd ?? null,
387 };
388 }
File History 2 commits
sha256:65ccb454656ea5acdea0a10e559b78bcde1eb6ff753ecc2911bc99d1c3d7cadd feat(calendar): enforce agent context tiers in retrieval AP… Human minor ⚠ 1 day ago
sha256:9103f98c89257ed2b01c237cea895dabb3e85ea337dccb1161c175e4422355b6 docs: accept Calendar Events v0 spec with Phase 0 security … Human 1 day ago