aaronrene / knowtation public
derived-artifact-storage-data-integrity.test.mjs
341 lines 13.7 KB
Raw
sha256:0d530f9ef27b8b75547d1db7701a74bc77b77aa8f3d7fa3a8672cf2af36e63bb reconcile: import GitHub-direct RBAC/OAuth/companion and ho… Human minor ⚠ breaking 7 hours ago
1 /**
2 * Tier 5 — DATA INTEGRITY: Phase 6 derived-artifact storage layer.
3 *
4 * Covers (§10 Data-integrity obligations):
5 * - Provenance round-trips intact and co-located with its artifact
6 * - No orphan after note delete (P6-g)
7 * - Crypto-shred: privacy_max ciphertext is unreadable after key destruction
8 * - Aggregate insight stale-flag + re-enrichment preserves source_event_id history
9 * - Re-enrichment never destroys the prior artifact on a failed gate
10 */
11
12 import { describe, it } from 'node:test';
13 import assert from 'node:assert/strict';
14
15 import {
16 createDerivedArtifactWriter,
17 WRITER_REASONS,
18 } from '../lib/companion-artifact-writer.mjs';
19
20 import {
21 createClientEncryptor,
22 } from '../lib/companion-client-encryptor.mjs';
23
24 import {
25 buildConvenienceProvenance,
26 PROVENANCE_SCHEMA_VERSION,
27 validateProvenance,
28 } from '../lib/companion-provenance-validator.mjs';
29
30 import { TERMINAL_STATES } from '../lib/companion-tier-resolver.mjs';
31
32 // ── Helpers ───────────────────────────────────────────────────────────────────
33
34 function buildIntegrityStores() {
35 const noteStore = new Map();
36 const vectorStore = new Map(); // notePath → point
37 const insightStore = [];
38 const maintenanceLog = [];
39
40 const writeNoteFn = (_vp, notePath, opts) => {
41 const prev = noteStore.get(notePath) ?? {};
42 noteStore.set(notePath, { ...prev, ...opts.frontmatter });
43 };
44
45 const vs = {
46 upsert: async (points) => {
47 for (const p of points) {
48 vectorStore.set(p.path, p);
49 }
50 },
51 deleteByPath: async (notePath) => {
52 vectorStore.delete(notePath);
53 },
54 };
55
56 const mm = {
57 store: (type, data) => {
58 if (type === 'insight') insightStore.push(data);
59 if (type === 'maintenance') maintenanceLog.push(data);
60 return { id: `mem_${Math.random().toString(36).slice(2)}`, ts: new Date().toISOString() };
61 },
62 };
63
64 return { noteStore, vectorStore, insightStore, maintenanceLog, writeNoteFn, vs, mm };
65 }
66
67 function selfCtx() {
68 return {
69 lane: 'local', containsPrivateData: false, isDelegate: false,
70 delegatedManagedAllowed: false, enrichesDelegatedPartition: false, delegatedEnrichmentAllowed: false,
71 };
72 }
73
74 // ── Provenance round-trip ─────────────────────────────────────────────────────
75
76 describe('data-integrity — provenance round-trips intact (D6.2.4)', () => {
77 it('ai_summary provenance sidecar contains all required fields after write', async () => {
78 const { noteStore, writeNoteFn } = buildIntegrityStores();
79 const writer = createDerivedArtifactWriter({ writeNoteFn, vaultPath: '/vault' });
80
81 const originalProv = buildConvenienceProvenance({
82 generatedBy: 'integrity-user',
83 source: 'companion',
84 model: 'integrity-model',
85 modelVersion: '2.0',
86 runtimeVersion: '1.5.0',
87 lane: 'local',
88 artifactType: 'ai_summary',
89 sourceNotePath: 'integrity/note.md',
90 sourceEventId: 'mem_int_001',
91 });
92
93 await writer.write({ summary: 'Integrity check summary.' }, originalProv, selfCtx());
94
95 const fm = noteStore.get('integrity/note.md');
96 const stored = fm.ai_summary_provenance;
97
98 // All D6.2.1 required fields must survive the round-trip
99 assert.equal(stored.generated_by, 'integrity-user');
100 assert.equal(stored.source, 'companion');
101 assert.equal(stored.model, 'integrity-model');
102 assert.equal(stored.model_version, '2.0');
103 assert.equal(stored.runtime_version, '1.5.0');
104 assert.equal(stored.lane, 'local');
105 assert.equal(stored.privacy_tier, 'convenience');
106 assert.equal(stored.source_note_path, 'integrity/note.md');
107 assert.equal(stored.source_event_id, 'mem_int_001');
108 assert.ok(stored.created_at);
109 assert.equal(stored.artifact_type, 'ai_summary');
110 assert.equal(stored.schema_version, PROVENANCE_SCHEMA_VERSION);
111 });
112
113 it('insight provenance round-trips with array source_event_id', async () => {
114 const { insightStore, writeNoteFn, mm } = buildIntegrityStores();
115 const writer = createDerivedArtifactWriter({ writeNoteFn, vaultPath: '/vault', mm });
116
117 const prov = buildConvenienceProvenance({
118 generatedBy: 'discover-pass',
119 source: 'companion',
120 model: 'insight-model',
121 modelVersion: '1.0',
122 lane: 'local',
123 artifactType: 'insight',
124 sourceNotePath: null,
125 sourceEventId: ['mem_c001', 'mem_c002', 'mem_c003'],
126 });
127
128 await writer.write(
129 { connections: ['A→B'], contradictions: [], open_questions: [], topic_count: 3 },
130 prov,
131 selfCtx(),
132 );
133
134 const stored = insightStore[0];
135 assert.ok(stored.provenance, 'Provenance must be co-located with the insight artifact');
136 assert.deepEqual(stored.provenance.source_event_id, ['mem_c001', 'mem_c002', 'mem_c003']);
137 assert.equal(stored.provenance.artifact_type, 'insight');
138 });
139 });
140
141 // ── No orphan after note delete (P6-g) ────────────────────────────────────────
142
143 describe('data-integrity — no orphan artifacts after delete (P6-g, D6.5.1)', () => {
144 it('note delete nulls ai_summary and purges vector entry', async () => {
145 const { noteStore, vectorStore, writeNoteFn, vs, mm } = buildIntegrityStores();
146 const writer = createDerivedArtifactWriter({
147 writeNoteFn, vaultPath: '/vault', vectorStore: vs, mm,
148 });
149
150 // Write summary
151 const sumprov = buildConvenienceProvenance({
152 generatedBy: 'u', source: 'companion', model: 'm', modelVersion: '1',
153 lane: 'local', artifactType: 'ai_summary',
154 sourceNotePath: 'orphan/note.md', sourceEventId: 'mem_o001',
155 });
156 await writer.write({ summary: 'Orphan check' }, sumprov, selfCtx());
157
158 // Write embedding
159 const embprov = buildConvenienceProvenance({
160 generatedBy: 'u', source: 'companion', model: 'm', modelVersion: '1',
161 lane: 'local', artifactType: 'embedding',
162 sourceNotePath: 'orphan/note.md', sourceEventId: 'mem_o002',
163 });
164 await writer.write({ vector: [1, 2, 3], payload: {} }, embprov, selfCtx());
165
166 // Verify pre-delete state
167 assert.ok(noteStore.has('orphan/note.md'));
168 assert.ok(vectorStore.has('orphan/note.md'));
169
170 // Delete
171 const delResult = await writer.deleteArtifacts({ notePath: 'orphan/note.md' });
172 assert.equal(delResult.ok, true);
173
174 // ai_summary nulled — no orphan
175 const fm = noteStore.get('orphan/note.md');
176 assert.equal(fm.ai_summary, null);
177 assert.equal(fm.ai_summary_provenance, null);
178
179 // Vector purged — no orphan
180 assert.equal(vectorStore.has('orphan/note.md'), false, 'Vector entry must be purged');
181 });
182
183 it('multiple sequential deletes are idempotent (second delete does not error)', async () => {
184 const { writeNoteFn, vs, mm } = buildIntegrityStores();
185 const writer = createDerivedArtifactWriter({
186 writeNoteFn, vaultPath: '/vault', vectorStore: vs, mm,
187 });
188
189 const r1 = await writer.deleteArtifacts({ notePath: 'nonexistent/note.md' });
190 // First delete: writeNoteFn and mm.store succeed (null writes are ok); vs.deleteByPath is fine
191 // The key assertion: it must not throw
192 assert.ok(r1.ok === true || r1.ok === false, 'Must return a result object');
193 });
194 });
195
196 // ── Crypto-shred: ciphertext is unreadable after key destruction ──────────────
197
198 describe('data-integrity — crypto-shred for privacy_max (D6.5.3)', () => {
199 it('stored ciphertext is not the plaintext (key destruction simulation)', async () => {
200 const { noteStore, writeNoteFn } = buildIntegrityStores();
201
202 // Simulate a user-held key that we then "destroy"
203 let keyAvailable = true;
204 const encryptor = createClientEncryptor({
205 isAvailable: () => keyAvailable,
206 encrypt: (bytes, _opts) => {
207 // XOR with 0xAA as a trivial "encryption"
208 const ct = new Uint8Array(bytes.length);
209 for (let i = 0; i < bytes.length; i++) ct[i] = bytes[i] ^ 0xaa;
210 return { ciphertext: ct, wrappedDekRef: 'dek-to-destroy', alg: 'STUB-XOR' };
211 },
212 });
213
214 const writer = createDerivedArtifactWriter({
215 writeNoteFn,
216 vaultPath: '/vault',
217 vaultRegistryAvailable: true,
218 encryptor,
219 });
220
221 const prov = buildConvenienceProvenance({
222 generatedBy: 'priv', source: 'companion', model: 'm', modelVersion: '1',
223 lane: 'local', artifactType: 'ai_summary',
224 sourceNotePath: 'private/shred.md', sourceEventId: 'mem_sh001',
225 });
226 const privProv = { ...prov, privacy_tier: 'privacy_max' };
227
228 const plaintext = 'My very private content that must never be host-readable.';
229 await writer.write({ summary: plaintext }, privProv, {
230 lane: 'local', containsPrivateData: true, isDelegate: false,
231 delegatedManagedAllowed: false,
232 });
233
234 const fm = noteStore.get('private/shred.md');
235 assert.ok(fm.ai_summary_ciphertext, 'Ciphertext must be stored');
236
237 // Simulate key destruction: wrappedDekRef recorded, key no longer accessible
238 keyAvailable = false;
239 assert.equal(encryptor.isAvailable('privacy_max', 'vault'), false);
240
241 // The stored ciphertext is not the plaintext
242 const storedBase64 = fm.ai_summary_ciphertext;
243 const decoded = Buffer.from(storedBase64, 'base64').toString('utf8');
244 assert.notEqual(decoded, plaintext, 'Stored bytes must not be plaintext');
245
246 // wrappedDekRef is recorded for targeted key destruction
247 assert.equal(fm.ai_summary_provenance.wrapped_dek_ref, 'dek-to-destroy');
248 });
249 });
250
251 // ── Stale-flag + re-enrichment preserves source_event_id history ──────────────
252
253 describe('data-integrity — stale-flag preserves source_event_id (D6.5.2, D6.7)', () => {
254 it('deleteArtifacts stores maintenance event with deleted_note_path', async () => {
255 const { maintenanceLog, writeNoteFn, mm } = buildIntegrityStores();
256 const writer = createDerivedArtifactWriter({
257 writeNoteFn, vaultPath: '/vault', mm,
258 });
259
260 await writer.deleteArtifacts({ notePath: 'agg/source.md' });
261
262 const maint = maintenanceLog.find((m) => m.deleted_note_path === 'agg/source.md');
263 assert.ok(maint, 'Maintenance stale-flag event must be recorded');
264 assert.equal(maint.action, 'artifact_delete_requested');
265 // source_event_id in the aggregate insight is preserved — the insight itself is NOT deleted
266 });
267 });
268
269 // ── Re-enrichment does not destroy prior artifact on failed gate ──────────────
270
271 describe('data-integrity — re-enrichment failure preserves prior artifact (D6.7 §fail-closed)', () => {
272 it('failed write on re-enrichment leaves original artifact unchanged', async () => {
273 const { noteStore, writeNoteFn } = buildIntegrityStores();
274
275 // Step 1: write original artifact
276 const writer = createDerivedArtifactWriter({ writeNoteFn, vaultPath: '/vault' });
277
278 const origProv = buildConvenienceProvenance({
279 generatedBy: 'u', source: 'companion', model: 'm', modelVersion: 'v1',
280 lane: 'local', artifactType: 'ai_summary',
281 sourceNotePath: 'reenrich/note.md', sourceEventId: 'mem_r001',
282 });
283 await writer.write({ summary: 'Original summary v1' }, origProv, selfCtx());
284
285 const origFm = { ...noteStore.get('reenrich/note.md') };
286 assert.equal(origFm.ai_summary, 'Original summary v1');
287
288 // Step 2: attempt re-enrichment with a broken writer (provenance missing)
289 const badProv = { ...origProv, generated_by: '' }; // invalid — will fail validation
290 const result = await writer.write({ summary: 'New summary v2' }, badProv, selfCtx());
291
292 assert.equal(result.ok, false);
293
294 // Original artifact MUST be unchanged (no destructive half-write)
295 const afterFm = noteStore.get('reenrich/note.md');
296 assert.equal(afterFm.ai_summary, 'Original summary v1', 'Prior artifact must be preserved on failed gate');
297 });
298
299 it('failed re-enrichment at consent gate leaves original unchanged', async () => {
300 const { noteStore, writeNoteFn } = buildIntegrityStores();
301 const writer = createDerivedArtifactWriter({ writeNoteFn, vaultPath: '/vault' });
302
303 const origProv = buildConvenienceProvenance({
304 generatedBy: 'u', source: 'companion', model: 'm', modelVersion: 'v1',
305 lane: 'local', artifactType: 'ai_summary',
306 sourceNotePath: 'reenrich2/note.md', sourceEventId: 'mem_r002',
307 });
308 await writer.write({ summary: 'Original v1' }, origProv, selfCtx());
309
310 const origFm = { ...noteStore.get('reenrich2/note.md') };
311
312 // Re-enrichment attempt that fails at consent (delegated without permission)
313 const result = await writer.write(
314 { summary: 'v2 attempt' },
315 origProv,
316 {
317 lane: 'local', containsPrivateData: false,
318 isDelegate: true, delegatedManagedAllowed: false,
319 enrichesDelegatedPartition: true, delegatedEnrichmentAllowed: false,
320 },
321 );
322
323 assert.equal(result.ok, false);
324 const afterFm = noteStore.get('reenrich2/note.md');
325 assert.equal(afterFm.ai_summary, 'Original v1', 'Consent-denied re-enrichment must leave original intact');
326 });
327 });
328
329 // ── Provenance cannot be forged or omitted (P6-e) ────────────────────────────
330
331 describe('data-integrity — provenance cannot be omitted (P6-e, D6.2)', () => {
332 it('write with missing provenance always fails', async () => {
333 const { writeNoteFn } = buildIntegrityStores();
334 const writer = createDerivedArtifactWriter({ writeNoteFn, vaultPath: '/vault' });
335
336 for (const badProv of [null, undefined, {}, { generated_by: 'u' }]) {
337 const r = await writer.write({ summary: 'test' }, badProv, selfCtx());
338 assert.equal(r.ok, false, `Provenance=${JSON.stringify(badProv)} must be rejected`);
339 }
340 });
341 });
File History 1 commit
sha256:0d530f9ef27b8b75547d1db7701a74bc77b77aa8f3d7fa3a8672cf2af36e63bb reconcile: import GitHub-direct RBAC/OAuth/companion and ho… Human minor ⚠ 7 hours ago