aaronrene / knowtation public
derived-artifact-storage-integration.test.mjs
395 lines 14.1 KB
Raw
sha256:0d530f9ef27b8b75547d1db7701a74bc77b77aa8f3d7fa3a8672cf2af36e63bb reconcile: import GitHub-direct RBAC/OAuth/companion and ho… Human minor ⚠ breaking 6 hours ago
1 /**
2 * Tier 2 — INTEGRATION: Phase 6 derived-artifact storage layer.
3 *
4 * Covers (§10 Integration obligations):
5 * - Writer pipeline end-to-end: validate→resolve→consent→encrypt→store
6 * across ai_summary, embedding, and insight artifact types
7 * - Migrated enrichIndexedNotes routes through the writer
8 * - Migrated runDiscoverPass routes through the writer
9 * - Convenience write lands host-readable with provenance (no ciphertext)
10 * - privacy_max write with unavailable encryptor → no plaintext fallback
11 * - deleteArtifacts removes from all stores for a note
12 */
13
14 import { describe, it, beforeEach } from 'node:test';
15 import assert from 'node:assert/strict';
16
17 import {
18 createDerivedArtifactWriter,
19 WRITER_REASONS,
20 } from '../lib/companion-artifact-writer.mjs';
21
22 import {
23 UNAVAILABLE_CLIENT_ENCRYPTOR,
24 createClientEncryptor,
25 ENCRYPTOR_REASONS,
26 } from '../lib/companion-client-encryptor.mjs';
27
28 import {
29 TERMINAL_STATES,
30 } from '../lib/companion-tier-resolver.mjs';
31
32 import {
33 buildConvenienceProvenance,
34 PROVENANCE_SCHEMA_VERSION,
35 } from '../lib/companion-provenance-validator.mjs';
36
37 import { runDiscoverPass } from '../lib/memory-consolidate.mjs';
38
39 // ── Test doubles ──────────────────────────────────────────────────────────────
40
41 function makeStores() {
42 const frontmatter = new Map();
43 const vectors = [];
44 const insights = [];
45 const maintenance = [];
46
47 const writeNoteFn = (_vaultPath, notePath, opts) => {
48 const prev = frontmatter.get(notePath) ?? {};
49 frontmatter.set(notePath, { ...prev, ...opts.frontmatter });
50 };
51
52 const vectorStore = {
53 upsert: async (points) => { vectors.push(...points); },
54 deleteByPath: async (notePath) => {
55 const idx = vectors.findIndex((v) => v.path === notePath);
56 if (idx >= 0) vectors.splice(idx, 1);
57 },
58 };
59
60 const mm = {
61 store: (type, data) => {
62 if (type === 'insight') insights.push(data);
63 if (type === 'maintenance') maintenance.push(data);
64 return { id: `mem_${Date.now()}`, ts: new Date().toISOString() };
65 },
66 };
67
68 return { frontmatter, vectors, insights, maintenance, writeNoteFn, vectorStore, mm };
69 }
70
71 function convenienceContext() {
72 return {
73 lane: 'local',
74 containsPrivateData: false,
75 isDelegate: false,
76 delegatedManagedAllowed: false,
77 enrichesDelegatedPartition: false,
78 delegatedEnrichmentAllowed: false,
79 };
80 }
81
82 function summaryProvenance(overrides = {}) {
83 return buildConvenienceProvenance({
84 generatedBy: 'user-abc',
85 source: 'companion',
86 model: 'llama-3',
87 modelVersion: '3.1',
88 lane: 'local',
89 artifactType: 'ai_summary',
90 sourceNotePath: 'notes/test.md',
91 sourceEventId: 'mem_001',
92 ...overrides,
93 });
94 }
95
96 function insightProvenance(overrides = {}) {
97 return buildConvenienceProvenance({
98 generatedBy: 'user-abc',
99 source: 'companion',
100 model: 'llama-3',
101 modelVersion: '3.1',
102 lane: 'local',
103 artifactType: 'insight',
104 sourceNotePath: null,
105 sourceEventId: ['mem_001', 'mem_002'],
106 ...overrides,
107 });
108 }
109
110 // ── Pipeline end-to-end: ai_summary ──────────────────────────────────────────
111
112 describe('writer pipeline — ai_summary, convenience tier', () => {
113 it('stores summary + provenance sidecar in note frontmatter', async () => {
114 const { frontmatter, writeNoteFn } = makeStores();
115 const w = createDerivedArtifactWriter({
116 writeNoteFn,
117 vaultPath: '/vault',
118 });
119
120 const artifact = { summary: 'This note discusses quantum entanglement.' };
121 const prov = summaryProvenance();
122 const r = await w.write(artifact, prov, convenienceContext());
123
124 assert.equal(r.ok, true);
125 assert.equal(r.terminalState, TERMINAL_STATES.HOST_READABLE);
126
127 const fm = frontmatter.get('notes/test.md');
128 assert.ok(fm, 'Frontmatter should be written');
129 assert.equal(fm.ai_summary, artifact.summary);
130 assert.ok(fm.ai_summary_provenance, 'Provenance sidecar should be written');
131 assert.equal(fm.ai_summary_provenance.artifact_type, 'ai_summary');
132 assert.equal(fm.ai_summary_provenance.privacy_tier, 'convenience');
133 // No ciphertext at convenience tier
134 assert.equal(fm.ai_summary_ciphertext, undefined);
135 });
136
137 it('provenance sidecar contains only safe fields (no secrets)', async () => {
138 const { frontmatter, writeNoteFn } = makeStores();
139 const w = createDerivedArtifactWriter({ writeNoteFn, vaultPath: '/vault' });
140 await w.write({ summary: 'fine' }, summaryProvenance(), convenienceContext());
141 const fm = frontmatter.get('notes/test.md');
142 const prov = fm.ai_summary_provenance;
143 // Must not contain key material
144 for (const key of Object.keys(prov)) {
145 const lk = key.toLowerCase();
146 assert.ok(
147 !lk.includes('secret') && !lk.includes('token') && !lk.includes('key') && !lk.includes('password'),
148 `Provenance sidecar must not contain sensitive key: ${key}`,
149 );
150 }
151 });
152 });
153
154 // ── Pipeline end-to-end: embedding ───────────────────────────────────────────
155
156 describe('writer pipeline — embedding, convenience tier', () => {
157 it('upserts vector with provenance metadata', async () => {
158 const { vectors, writeNoteFn, vectorStore } = makeStores();
159 const w = createDerivedArtifactWriter({
160 writeNoteFn,
161 vaultPath: '/vault',
162 vectorStore,
163 });
164
165 const prov = buildConvenienceProvenance({
166 generatedBy: 'user-abc',
167 source: 'companion',
168 model: 'embed-model',
169 modelVersion: '1.0',
170 lane: 'local',
171 artifactType: 'embedding',
172 sourceNotePath: 'notes/embed.md',
173 sourceEventId: 'mem_emb_001',
174 });
175 const artifact = {
176 vector: [0.1, 0.2, 0.3],
177 payload: { path: 'notes/embed.md' },
178 };
179 const r = await w.write(artifact, prov, convenienceContext());
180
181 assert.equal(r.ok, true);
182 assert.equal(r.terminalState, TERMINAL_STATES.HOST_READABLE);
183 assert.equal(vectors.length, 1);
184 assert.deepEqual(vectors[0].vector, [0.1, 0.2, 0.3]);
185 assert.ok(vectors[0].payload.provenance);
186 });
187 });
188
189 // ── Pipeline end-to-end: insight ─────────────────────────────────────────────
190
191 describe('writer pipeline — insight, convenience tier', () => {
192 it('stores insight with provenance via MemoryManager', async () => {
193 const { insights, writeNoteFn, mm } = makeStores();
194 const w = createDerivedArtifactWriter({
195 writeNoteFn,
196 vaultPath: '/vault',
197 mm,
198 });
199
200 const prov = insightProvenance();
201 const artifact = {
202 connections: ['A relates to B'],
203 contradictions: [],
204 open_questions: ['What is X?'],
205 topic_count: 2,
206 };
207 const r = await w.write(artifact, prov, convenienceContext());
208
209 assert.equal(r.ok, true);
210 assert.equal(insights.length, 1);
211 assert.deepEqual(insights[0].connections, ['A relates to B']);
212 assert.ok(insights[0].provenance);
213 });
214 });
215
216 // ── Privacy-max: fail-closed with unavailable encryptor ───────────────────────
217
218 describe('writer pipeline — privacy_max with unavailable encryptor', () => {
219 it('returns ENCRYPTION_UNAVAILABLE and writes nothing', async () => {
220 const { frontmatter, writeNoteFn } = makeStores();
221 const w = createDerivedArtifactWriter({
222 writeNoteFn,
223 vaultPath: '/vault',
224 vaultRegistryAvailable: true,
225 // No encryptor provided → defaults to UNAVAILABLE_CLIENT_ENCRYPTOR
226 });
227
228 // buildConvenienceProvenance always sets privacy_tier=convenience; override after
229 const prov = { ...summaryProvenance(), privacy_tier: 'privacy_max' };
230 const r = await w.write({ summary: 'private' }, prov, convenienceContext());
231
232 assert.equal(r.ok, false);
233 assert.equal(r.reason, WRITER_REASONS.ENCRYPTION_UNAVAILABLE);
234 assert.equal(frontmatter.size, 0, 'No frontmatter should be written');
235 });
236
237 it('never stores plaintext when privacy_max encryption fails', async () => {
238 const { writeNoteFn, frontmatter } = makeStores();
239 const failingEncryptor = createClientEncryptor({
240 isAvailable: () => true,
241 encrypt: () => { throw new Error('encrypt_failed'); },
242 });
243
244 const w = createDerivedArtifactWriter({
245 writeNoteFn,
246 vaultPath: '/vault',
247 vaultRegistryAvailable: true,
248 encryptor: failingEncryptor,
249 });
250
251 // Override privacy_tier to privacy_max after building convenience provenance
252 const prov = { ...summaryProvenance(), privacy_tier: 'privacy_max' };
253 const r = await w.write({ summary: 'private' }, prov, convenienceContext());
254
255 assert.equal(r.ok, false);
256 assert.equal(r.reason, WRITER_REASONS.ENCRYPTION_FAILED);
257 assert.equal(frontmatter.size, 0, 'No plaintext must ever be written on encryption failure');
258 });
259 });
260
261 // ── Privacy-max: success path with working encryptor ─────────────────────────
262
263 describe('writer pipeline — privacy_max with working encryptor', () => {
264 it('stores ciphertext + wrappedDekRef, never plaintext', async () => {
265 const { frontmatter, writeNoteFn } = makeStores();
266 const workingEncryptor = createClientEncryptor({
267 isAvailable: () => true,
268 encrypt: (bytes, _opts) => ({
269 ciphertext: new Uint8Array(Array.from(bytes).map((b) => b ^ 0xff)), // trivial XOR
270 wrappedDekRef: 'dek-ref-vault-001',
271 alg: 'X-TEST-256',
272 }),
273 });
274
275 const w = createDerivedArtifactWriter({
276 writeNoteFn,
277 vaultPath: '/vault',
278 vaultRegistryAvailable: true,
279 encryptor: workingEncryptor,
280 });
281
282 // Override privacy_tier after building (buildConvenienceProvenance always emits 'convenience')
283 const prov = { ...summaryProvenance(), privacy_tier: 'privacy_max' };
284 const r = await w.write({ summary: 'my private note' }, prov, convenienceContext());
285
286 assert.equal(r.ok, true);
287 assert.equal(r.terminalState, TERMINAL_STATES.CLIENT_ENCRYPTED);
288
289 const fm = frontmatter.get('notes/test.md');
290 assert.ok(fm.ai_summary_ciphertext, 'Should store ciphertext');
291 assert.equal(fm.ai_summary_provenance.wrapped_dek_ref, 'dek-ref-vault-001');
292 assert.equal(fm.ai_summary_provenance.alg, 'X-TEST-256');
293 // No plaintext ai_summary field
294 assert.equal(fm.ai_summary, undefined, 'Plaintext ai_summary must not be stored at privacy_max');
295 });
296 });
297
298 // ── deleteArtifacts — removes from all stores ─────────────────────────────────
299
300 describe('writer.deleteArtifacts — single-path deletion (D6.5.4)', () => {
301 it('removes frontmatter ai_summary, vector, and stale-flags insights', async () => {
302 const { frontmatter, vectors, maintenance, writeNoteFn, vectorStore, mm } = makeStores();
303
304 // Pre-populate
305 frontmatter.set('notes/del.md', { ai_summary: 'old', ai_summary_provenance: {} });
306 vectors.push({ path: 'notes/del.md', vector: [1, 2] });
307
308 const w = createDerivedArtifactWriter({ writeNoteFn, vaultPath: '/vault', vectorStore, mm });
309 const r = await w.deleteArtifacts({ notePath: 'notes/del.md' });
310
311 assert.equal(r.ok, true);
312
313 // Frontmatter nulled out
314 const fm = frontmatter.get('notes/del.md');
315 assert.equal(fm.ai_summary, null);
316 assert.equal(fm.ai_summary_provenance, null);
317
318 // Vector purged
319 assert.equal(vectors.filter((v) => v.path === 'notes/del.md').length, 0);
320
321 // Maintenance event for stale-flagging
322 assert.equal(maintenance.length, 1);
323 assert.equal(maintenance[0].deleted_note_path, 'notes/del.md');
324 });
325
326 it('reports partial failure but still succeeds on the working stores', async () => {
327 const { frontmatter, writeNoteFn, mm } = makeStores();
328 const badVectorStore = {
329 upsert: async () => {},
330 deleteByPath: async () => { throw new Error('qdrant down'); },
331 };
332
333 const w = createDerivedArtifactWriter({
334 writeNoteFn,
335 vaultPath: '/vault',
336 vectorStore: badVectorStore,
337 mm,
338 });
339
340 const r = await w.deleteArtifacts({ notePath: 'notes/del.md' });
341 // Partial: frontmatter and maintenance succeeded, vector failed
342 assert.equal(r.ok, false);
343 assert.ok(r.failed.includes('vector'));
344 });
345 });
346
347 // ── Migrated runDiscoverPass routes through writer ────────────────────────────
348
349 describe('runDiscoverPass (migrated) — routes insight through writer', () => {
350 it('calls writer.write instead of mm.store directly', async () => {
351 const writeCalls = [];
352 const mockWriter = {
353 write: async (artifact, provenance, context) => {
354 writeCalls.push({ artifact, provenance, context });
355 return { ok: true, terminalState: 'host_readable' };
356 },
357 deleteArtifacts: async () => ({ ok: true, stores: [] }),
358 checkReEnrichmentEligibility: () => ({ eligible: false, reason: 'current' }),
359 };
360
361 const mockMm = { store: () => ({ id: 'x', ts: '...' }) };
362
363 const config = {
364 vault_path: '/vault',
365 vault_id: 'test-vault',
366 llm: { model: 'test-model', model_version: '1.0' },
367 memory: {},
368 daemon: { llm: { max_tokens: 100 } },
369 };
370
371 const consolidations = [
372 { id: 'mem_c1', data: { topic: 'testing', facts: ['Fact A', 'Fact B'] } },
373 { id: 'mem_c2', data: { topic: 'learning', facts: ['Fact C'] } },
374 ];
375
376 const fakeLlm = async () => JSON.stringify({
377 connections: ['Testing relates to learning'],
378 contradictions: [],
379 open_questions: ['How?'],
380 });
381
382 await runDiscoverPass(config, consolidations, {
383 llmFn: fakeLlm,
384 mm: mockMm,
385 writer: mockWriter,
386 });
387
388 assert.equal(writeCalls.length, 1, 'writer.write must be called exactly once');
389 const { artifact, provenance } = writeCalls[0];
390 assert.ok(Array.isArray(artifact.connections));
391 assert.equal(provenance.artifact_type, 'insight');
392 assert.equal(provenance.privacy_tier, 'convenience');
393 assert.ok(Array.isArray(provenance.source_event_id));
394 });
395 });
File History 1 commit
sha256:0d530f9ef27b8b75547d1db7701a74bc77b77aa8f3d7fa3a8672cf2af36e63bb reconcile: import GitHub-direct RBAC/OAuth/companion and ho… Human minor ⚠ 6 hours ago