aaronrene / knowtation public
companion-runtime-manager-security.test.mjs
451 lines 19.2 KB
Raw
sha256:65ccb454656ea5acdea0a10e559b78bcde1eb6ff753ecc2911bc99d1c3d7cadd feat(calendar): enforce agent context tiers in retrieval AP… Human minor ⚠ breaking 1 day ago
1 /**
2 * Tier 7 — SECURITY: lib/companion-runtime-manager.mjs
3 *
4 * The security tier is the centerpiece of Phase 4 testing. It covers, at minimum:
5 *
6 * (a) SUPPLY-CHAIN: a wrong/missing integrity digest rejects the model — no execution on
7 * unverified bytes. An oversized or foreign-source download is rejected. HTTPS-only.
8 * (b) RESOURCE EXHAUSTION: backpressure trips at the configured bound. The lifecycle gate
9 * never serves inference in a non-ready state.
10 * (c) AMBIENT AUTHORITY: the module imports no vault, canister, keychain, or JWT handle.
11 * No secret, model path, URL, digest value, or access token appears in any reason
12 * string, return value, or thrown error.
13 * (d) CONSTANT-TIME: the integrity digest comparison is constant-time (no length/content
14 * timing oracle on the expected digest).
15 *
16 * Reference: docs/COMPANION-APP-PHASE-4-RUNTIME-MANAGER.md §2 (integrity), §3 (lifecycle),
17 * §4 (backpressure); docs/COMPANION-APP-DESIGN-AND-AUTHORIZATION-GATE.md §4.6
18 * (no ambient authority).
19 */
20
21 import { describe, it } from 'node:test';
22 import assert from 'node:assert/strict';
23 import crypto from 'node:crypto';
24
25 import {
26 RUNTIME_MANAGER_REASONS,
27 LIFECYCLE_STATES,
28 LIFECYCLE_EVENTS,
29 createIntegrityAccumulator,
30 verifyModelBytes,
31 validateSourceUrl,
32 validateIntegritySpec,
33 createLifecycleState,
34 transitionLifecycle,
35 canServeInference,
36 createAdmissionState,
37 evaluateAdmission,
38 recordInFlight,
39 createResourceLimits,
40 evaluateResourceLimits,
41 evaluateRuntimeRequest,
42 } from '../lib/companion-runtime-manager.mjs';
43
44 function makeDigest(data) {
45 return crypto.createHash('sha256').update(data).digest('hex');
46 }
47
48 const ALLOWED_URLS = ['https://models.example.com/'];
49 const VALID_URL = 'https://models.example.com/model.bin';
50 const VALID_DATA = Buffer.from('a legitimate model binary payload');
51 const VALID_DIGEST = makeDigest(VALID_DATA);
52 const VALID_SIZE = VALID_DATA.length;
53
54 const READY = { state: LIFECYCLE_STATES.READY };
55 const VALID_LIMITS = createResourceLimits({ maxRamBytes: 8e9, maxVramBytes: 4e9, maxCpuPercent: 80 });
56 const VALID_OBS = { ramBytes: 1e9, vramBytes: 0.5e9, cpuPercent: 10 };
57
58 // ── (a) SUPPLY-CHAIN INTEGRITY ────────────────────────────────────────────────
59
60 describe('security: supply-chain — wrong digest rejects model before execution', () => {
61 it('wrong digest (all zeros) is rejected — DIGEST_MISMATCH', () => {
62 const r = verifyModelBytes({
63 fileData: VALID_DATA, expectedDigest: '0'.repeat(64),
64 expectedSizeBytes: VALID_SIZE, sourceUrl: VALID_URL, allowedSourceUrls: ALLOWED_URLS,
65 });
66 assert.equal(r.ok, false);
67 assert.equal(r.reason, RUNTIME_MANAGER_REASONS.DIGEST_MISMATCH);
68 });
69
70 it('wrong digest (one hex char flipped) is rejected', () => {
71 const flipped = VALID_DIGEST.slice(0, -1) + (VALID_DIGEST.endsWith('0') ? '1' : '0');
72 const r = verifyModelBytes({
73 fileData: VALID_DATA, expectedDigest: flipped,
74 expectedSizeBytes: VALID_SIZE, sourceUrl: VALID_URL, allowedSourceUrls: ALLOWED_URLS,
75 });
76 assert.equal(r.ok, false);
77 assert.equal(r.reason, RUNTIME_MANAGER_REASONS.DIGEST_MISMATCH);
78 });
79
80 it('missing digest (empty string spec) is rejected before download starts', () => {
81 const r = validateIntegritySpec('', VALID_SIZE);
82 assert.equal(r.ok, false);
83 assert.equal(r.reason, RUNTIME_MANAGER_REASONS.MALFORMED_SPEC);
84 });
85
86 it('missing digest (null) is rejected', () => {
87 const r = validateIntegritySpec(null, VALID_SIZE);
88 assert.equal(r.ok, false);
89 });
90
91 it('accumulator: digest mismatch from correct-length corrupted data → DIGEST_MISMATCH', () => {
92 const acc = createIntegrityAccumulator({
93 expectedDigest: VALID_DIGEST, expectedSizeBytes: VALID_SIZE,
94 sourceUrl: VALID_URL, allowedSourceUrls: ALLOWED_URLS,
95 });
96 const corrupted = Buffer.from(VALID_DATA);
97 corrupted[0] ^= 0x01; // flip one bit
98 acc.update(corrupted);
99 const verdict = acc.finalize();
100 assert.equal(verdict.ok, false);
101 assert.equal(verdict.reason, RUNTIME_MANAGER_REASONS.DIGEST_MISMATCH);
102 });
103
104 it('accumulator: size mismatch (one extra byte appended) → SIZE_MISMATCH', () => {
105 const acc = createIntegrityAccumulator({
106 expectedDigest: VALID_DIGEST, expectedSizeBytes: VALID_SIZE,
107 sourceUrl: VALID_URL, allowedSourceUrls: ALLOWED_URLS,
108 });
109 acc.update(VALID_DATA);
110 acc.update(Buffer.from([0x00])); // one extra byte
111 const verdict = acc.finalize();
112 assert.equal(verdict.ok, false);
113 assert.equal(verdict.reason, RUNTIME_MANAGER_REASONS.SIZE_MISMATCH);
114 });
115
116 it('accumulator: oversized download rejected when expectedSizeBytes is correct but more arrives', () => {
117 // An attacker delivers more bytes than declared — size mismatch
118 const declared = VALID_SIZE;
119 const acc = createIntegrityAccumulator({
120 expectedDigest: VALID_DIGEST, expectedSizeBytes: declared,
121 sourceUrl: VALID_URL, allowedSourceUrls: ALLOWED_URLS,
122 });
123 // Feed declared + 1000 extra bytes
124 acc.update(VALID_DATA);
125 acc.update(Buffer.alloc(1000, 0x42)); // extra bytes from attacker
126 const verdict = acc.finalize();
127 assert.equal(verdict.ok, false);
128 assert.equal(verdict.reason, RUNTIME_MANAGER_REASONS.SIZE_MISMATCH);
129 });
130
131 it('HTTP source URL is rejected at spec-validation time (not download time)', () => {
132 const r = validateSourceUrl('http://models.example.com/model.bin', ALLOWED_URLS);
133 assert.equal(r.ok, false);
134 assert.equal(r.reason, RUNTIME_MANAGER_REASONS.SCHEME_NOT_ALLOWED);
135 });
136
137 it('foreign source URL (not in allowlist) is rejected', () => {
138 const r = validateSourceUrl('https://evil-models.net/model.bin', ALLOWED_URLS);
139 assert.equal(r.ok, false);
140 assert.equal(r.reason, RUNTIME_MANAGER_REASONS.SOURCE_NOT_ALLOWED);
141 });
142
143 it('empty allowlist rejects any URL — fail-closed', () => {
144 const r = validateSourceUrl(VALID_URL, []);
145 assert.equal(r.ok, false);
146 assert.equal(r.reason, RUNTIME_MANAGER_REASONS.SOURCE_NOT_ALLOWED);
147 });
148
149 it('accumulator throws when constructed with HTTP source URL', () => {
150 assert.throws(
151 () => createIntegrityAccumulator({
152 expectedDigest: VALID_DIGEST, expectedSizeBytes: VALID_SIZE,
153 sourceUrl: 'http://models.example.com/model.bin', allowedSourceUrls: ALLOWED_URLS,
154 }),
155 (err) => {
156 assert.ok(err instanceof TypeError);
157 // Error message must NOT contain the actual URL or digest (no secret leak)
158 assert.ok(!err.message.includes('http://models.example.com'), 'URL leaked in error');
159 return true;
160 },
161 );
162 });
163
164 it('accumulator throws when constructed with foreign source URL not in allowlist', () => {
165 assert.throws(
166 () => createIntegrityAccumulator({
167 expectedDigest: VALID_DIGEST, expectedSizeBytes: VALID_SIZE,
168 sourceUrl: 'https://evil-models.net/model.bin', allowedSourceUrls: ALLOWED_URLS,
169 }),
170 (err) => {
171 assert.ok(err instanceof TypeError);
172 assert.ok(!err.message.includes('evil-models.net'), 'URL leaked in error');
173 return true;
174 },
175 );
176 });
177 });
178
179 // ── (a) SUPPLY-CHAIN — no execution on unverified bytes ──────────────────────
180
181 describe('security: no execution path possible without integrity passing', () => {
182 it('lifecycle gate can only reach ready via health_ok after start', () => {
183 // There is no direct "stopped → ready" transition
184 const r = transitionLifecycle(createLifecycleState(), LIFECYCLE_EVENTS.HEALTH_OK);
185 assert.equal(r.ok, false); // invalid transition from stopped
186
187 // Starting → ready requires health_ok, not any other event
188 const s = { state: LIFECYCLE_STATES.STARTING };
189 const r2 = transitionLifecycle(s, LIFECYCLE_EVENTS.DRAIN);
190 assert.equal(r2.ok, false);
191 const r3 = transitionLifecycle(s, LIFECYCLE_EVENTS.STOPPED);
192 assert.equal(r3.ok, false);
193 const r4 = transitionLifecycle(s, LIFECYCLE_EVENTS.START);
194 assert.equal(r4.ok, false);
195 });
196
197 it('canServeInference is false in all non-ready states', () => {
198 assert.equal(canServeInference({ state: LIFECYCLE_STATES.STOPPED }), false);
199 assert.equal(canServeInference({ state: LIFECYCLE_STATES.STARTING }), false);
200 assert.equal(canServeInference({ state: LIFECYCLE_STATES.DRAINING }), false);
201 assert.equal(canServeInference({ state: LIFECYCLE_STATES.READY }), true); // only this
202 });
203 });
204
205 // ── (b) RESOURCE EXHAUSTION — backpressure ────────────────────────────────────
206
207 describe('security: backpressure — flood of requests hits bound, never OOM overflow', () => {
208 it('backpressure trips at exact maxInFlight boundary (100 requests)', () => {
209 const MAX = 100;
210 let admission = createAdmissionState({ maxInFlight: MAX, queueBound: 50 });
211
212 for (let i = 0; i < MAX; i++) admission = recordInFlight(admission);
213
214 // Exactly at the boundary — next request is AT_CAPACITY
215 const r = evaluateAdmission(admission);
216 assert.equal(r.ok, false);
217 assert.equal(r.reason, RUNTIME_MANAGER_REASONS.AT_CAPACITY);
218 assert.equal(admission.inFlight, MAX); // exact count, no overflow
219 });
220
221 it('queue_full trips at exact queueBound — third layer of defence', () => {
222 const MAX_F = 2, MAX_Q = 5;
223 let admission = createAdmissionState({ maxInFlight: MAX_F, queueBound: MAX_Q });
224 for (let i = 0; i < MAX_F; i++) admission = recordInFlight(admission);
225 admission = { ...admission, queued: MAX_Q };
226
227 const r = evaluateAdmission(admission);
228 assert.equal(r.ok, false);
229 assert.equal(r.reason, RUNTIME_MANAGER_REASONS.QUEUE_FULL);
230 });
231
232 it('evaluateRuntimeRequest blocks when lifecycle is not ready (flood cannot bypass)', () => {
233 const floodAdmission = createAdmissionState({ maxInFlight: 1000, queueBound: 1000 });
234 const stopped = createLifecycleState();
235 // Even with unlimited admission capacity, lifecycle blocks all
236 for (let i = 0; i < 1000; i++) {
237 const r = evaluateRuntimeRequest({
238 lifecycleState: stopped, admissionState: floodAdmission,
239 resourceObservation: VALID_OBS, resourceLimits: VALID_LIMITS,
240 });
241 assert.equal(r.ok, false);
242 assert.equal(r.reason, RUNTIME_MANAGER_REASONS.NOT_READY);
243 }
244 });
245 });
246
247 // ── (b) RESOURCE EXHAUSTION — over-limit rejection ───────────────────────────
248
249 describe('security: resource limits prevent OOM', () => {
250 it('RAM over limit → rejected before inference', () => {
251 const obs = { ...VALID_OBS, ramBytes: VALID_LIMITS.maxRamBytes + 1 };
252 const r = evaluateRuntimeRequest({
253 lifecycleState: READY, admissionState: createAdmissionState({ maxInFlight: 10, queueBound: 20 }),
254 resourceObservation: obs, resourceLimits: VALID_LIMITS,
255 });
256 assert.equal(r.ok, false);
257 assert.equal(r.reason, RUNTIME_MANAGER_REASONS.RAM_OVER_LIMIT);
258 });
259
260 it('VRAM over limit → rejected before inference', () => {
261 const obs = { ...VALID_OBS, vramBytes: VALID_LIMITS.maxVramBytes + 1 };
262 const r = evaluateRuntimeRequest({
263 lifecycleState: READY, admissionState: createAdmissionState({ maxInFlight: 10, queueBound: 20 }),
264 resourceObservation: obs, resourceLimits: VALID_LIMITS,
265 });
266 assert.equal(r.ok, false);
267 assert.equal(r.reason, RUNTIME_MANAGER_REASONS.VRAM_OVER_LIMIT);
268 });
269
270 it('CPU over limit → rejected before inference', () => {
271 const obs = { ...VALID_OBS, cpuPercent: VALID_LIMITS.maxCpuPercent + 1 };
272 const r = evaluateRuntimeRequest({
273 lifecycleState: READY, admissionState: createAdmissionState({ maxInFlight: 10, queueBound: 20 }),
274 resourceObservation: obs, resourceLimits: VALID_LIMITS,
275 });
276 assert.equal(r.ok, false);
277 assert.equal(r.reason, RUNTIME_MANAGER_REASONS.CPU_OVER_LIMIT);
278 });
279 });
280
281 // ── (c) AMBIENT AUTHORITY — no vault/JWT/canister in module exports ───────────
282
283 describe('security: no ambient authority in module exports', () => {
284 it('module exports do not include any vault, keychain, canister, or JWT accessor', async () => {
285 const mod = await import('../lib/companion-runtime-manager.mjs');
286 const exports = Object.keys(mod);
287 const forbidden = ['vault', 'keychain', 'jwt', 'token', 'canister', 'session', 'auth', 'secret'];
288 for (const key of exports) {
289 for (const bad of forbidden) {
290 assert.ok(
291 !key.toLowerCase().includes(bad),
292 `Export "${key}" looks like it exposes a sensitive authority (contains "${bad}")`,
293 );
294 }
295 }
296 });
297
298 it('evaluateRuntimeRequest returns a verdict with ok and reason only — no embedded secrets', () => {
299 const r = evaluateRuntimeRequest({
300 lifecycleState: READY,
301 admissionState: createAdmissionState({ maxInFlight: 4, queueBound: 8 }),
302 resourceObservation: VALID_OBS,
303 resourceLimits: VALID_LIMITS,
304 });
305 // Must have exactly ok and reason; no extra properties carrying data
306 const keys = Object.keys(r);
307 assert.deepEqual(keys.sort(), ['ok', 'reason'].sort());
308 assert.equal(typeof r.ok, 'boolean');
309 assert.equal(typeof r.reason, 'string');
310 });
311 });
312
313 // ── (c) NO SECRET IN REASON STRINGS ──────────────────────────────────────────
314
315 describe('security: no secret/path/URL/digest in reason strings', () => {
316 const secretPatterns = [
317 VALID_URL,
318 VALID_DIGEST,
319 'models.example.com',
320 'https://',
321 'http://',
322 '/model.bin',
323 ];
324
325 function assertNoSecretInReason(r, label) {
326 for (const pattern of secretPatterns) {
327 assert.ok(
328 !r.reason.includes(pattern),
329 `Reason "${r.reason}" contains secret pattern "${pattern}" in test: ${label}`,
330 );
331 }
332 }
333
334 it('validateSourceUrl failure reasons contain no URL', () => {
335 assertNoSecretInReason(validateSourceUrl('http://models.example.com/m', ALLOWED_URLS), 'http src');
336 assertNoSecretInReason(validateSourceUrl('https://evil.net/m', ALLOWED_URLS), 'foreign src');
337 assertNoSecretInReason(validateSourceUrl('bad url', ALLOWED_URLS), 'bad url');
338 });
339
340 it('verifyModelBytes failure reasons contain no digest or URL', () => {
341 assertNoSecretInReason(
342 verifyModelBytes({ fileData: VALID_DATA, expectedDigest: '0'.repeat(64), expectedSizeBytes: VALID_SIZE, sourceUrl: VALID_URL, allowedSourceUrls: ALLOWED_URLS }),
343 'wrong digest',
344 );
345 assertNoSecretInReason(
346 verifyModelBytes({ fileData: VALID_DATA, expectedDigest: VALID_DIGEST, expectedSizeBytes: VALID_SIZE + 1, sourceUrl: VALID_URL, allowedSourceUrls: ALLOWED_URLS }),
347 'size mismatch',
348 );
349 });
350
351 it('evaluateRuntimeRequest failure reasons contain no observation values', () => {
352 const obs = { ...VALID_OBS, ramBytes: VALID_LIMITS.maxRamBytes + 1 };
353 const r = evaluateRuntimeRequest({
354 lifecycleState: READY, admissionState: createAdmissionState({ maxInFlight: 4, queueBound: 8 }),
355 resourceObservation: obs, resourceLimits: VALID_LIMITS,
356 });
357 // Reason must not contain numeric values from the observation
358 assert.ok(!r.reason.includes(String(obs.ramBytes)), 'ramBytes leaked in reason');
359 });
360
361 it('accumulator finalize failure reasons are fixed constants', () => {
362 const acc = createIntegrityAccumulator({
363 expectedDigest: VALID_DIGEST, expectedSizeBytes: VALID_SIZE,
364 sourceUrl: VALID_URL, allowedSourceUrls: ALLOWED_URLS,
365 });
366 acc.update(Buffer.from('x'.repeat(VALID_SIZE))); // wrong content
367 const verdict = acc.finalize();
368 assertNoSecretInReason(verdict, 'digest mismatch');
369 });
370 });
371
372 // ── (d) CONSTANT-TIME DIGEST COMPARISON ──────────────────────────────────────
373
374 describe('security: constant-time digest comparison in integrity accumulator', () => {
375 it('timing is not correlated with prefix match length (statistical bound)', () => {
376 // We measure timing for: all-zeros wrong digest vs. correct-prefix-but-wrong-suffix digest.
377 // Both should produce DIGEST_MISMATCH in similar time. This is a statistical test;
378 // the hash-of-hash approach in the implementation makes the comparison truly constant-time.
379 const N = 100;
380
381 // Prefix matches perfectly for many chars then diverges
382 const prefixWrong = VALID_DIGEST.slice(0, 60) + '0000';
383 // Completely different digest
384 const allWrong = '0'.repeat(64);
385
386 const timeWith = (digest) => {
387 const start = performance.now();
388 for (let i = 0; i < N; i++) {
389 const acc = createIntegrityAccumulator({
390 expectedDigest: VALID_DIGEST, expectedSizeBytes: VALID_SIZE,
391 sourceUrl: VALID_URL, allowedSourceUrls: ALLOWED_URLS,
392 });
393 acc.update(VALID_DATA);
394 // Override internal state by creating a new acc with the attacker's digest as expected
395 // Actually test via verifyModelBytes which uses the same constant-time path
396 }
397 return performance.now() - start;
398 };
399
400 // Use verifyModelBytes which exposes the same comparison path
401 const timeVerify = (digest) => {
402 const start = performance.now();
403 for (let i = 0; i < N; i++) {
404 verifyModelBytes({
405 fileData: VALID_DATA, expectedDigest: digest, expectedSizeBytes: VALID_SIZE,
406 sourceUrl: VALID_URL, allowedSourceUrls: ALLOWED_URLS,
407 });
408 }
409 return performance.now() - start;
410 };
411
412 const tAll = timeVerify(allWrong);
413 const tPrefix = timeVerify(prefixWrong);
414 // Allow up to 5× difference — a true timing oracle would be 10-100×.
415 // This is a sanity check, not a rigorous timing test (which requires controlled hardware).
416 const ratio = Math.max(tAll, tPrefix) / Math.min(tAll, tPrefix);
417 assert.ok(ratio < 5, `Timing ratio ${ratio.toFixed(2)} suggests timing oracle. tAll=${tAll.toFixed(2)}ms tPrefix=${tPrefix.toFixed(2)}ms`);
418 });
419 });
420
421 // ── (c) FAIL-CLOSED POSTURE — any ambiguous/null input denies ─────────────────
422
423 describe('security: global fail-closed posture', () => {
424 const nullInputCases = [
425 { fn: () => evaluateRuntimeRequest(null), label: 'null params' },
426 { fn: () => evaluateRuntimeRequest({}), label: 'empty params' },
427 { fn: () => evaluateRuntimeRequest({ lifecycleState: null, admissionState: null, resourceObservation: null, resourceLimits: null }), label: 'all null fields' },
428 { fn: () => evaluateAdmission(null), label: 'null admission' },
429 { fn: () => evaluateAdmission(undefined), label: 'undefined admission' },
430 { fn: () => evaluateResourceLimits(null, VALID_LIMITS), label: 'null observation' },
431 { fn: () => evaluateResourceLimits(VALID_OBS, null), label: 'null limits' },
432 { fn: () => transitionLifecycle(null, LIFECYCLE_EVENTS.START), label: 'null lifecycle state' },
433 ];
434
435 for (const { fn, label } of nullInputCases) {
436 it(`${label} → deny (ok:false)`, () => {
437 const r = fn();
438 assert.equal(r.ok, false, `expected ok:false for: ${label}`);
439 });
440 }
441
442 it('evaluateRuntimeRequest never throws on any input', () => {
443 const throwingInputs = [null, undefined, 'string', 42, [], () => {}];
444 for (const inp of throwingInputs) {
445 assert.doesNotThrow(
446 () => evaluateRuntimeRequest(inp),
447 `threw on input: ${JSON.stringify(inp)}`,
448 );
449 }
450 });
451 });
File History 2 commits
sha256:65ccb454656ea5acdea0a10e559b78bcde1eb6ff753ecc2911bc99d1c3d7cadd feat(calendar): enforce agent context tiers in retrieval AP… Human minor ⚠ 1 day ago
sha256:9103f98c89257ed2b01c237cea895dabb3e85ea337dccb1161c175e4422355b6 docs: accept Calendar Events v0 spec with Phase 0 security … Human 1 day ago