aaronrene / knowtation public
llm-complete-openrouter-performance.test.mjs
89 lines 3.0 KB
Raw
sha256:65ccb454656ea5acdea0a10e559b78bcde1eb6ff753ecc2911bc99d1c3d7cadd feat(calendar): enforce agent context tiers in retrieval AP… Human minor ⚠ breaking 1 day ago
1 /**
2 * Tier 6 — PERFORMANCE: the OpenRouter lane must add negligible overhead around the network
3 * call and must not block the event loop.
4 *
5 * The HTTP boundary is mocked to return immediately, so what is measured is the lane's own
6 * per-call overhead (provider selection, header/body construction, JSON parse). Thresholds are
7 * deliberately generous to stay non-flaky on shared CI while still catching gross regressions
8 * (e.g. an accidental O(n^2) build or a synchronous sleep).
9 */
10 import { describe, it, beforeEach, afterEach } from 'node:test';
11 import assert from 'node:assert';
12 import { completeChat } from '../lib/llm-complete.mjs';
13
14 const ORIG = { ...process.env };
15 const origFetch = globalThis.fetch;
16
17 const CHAT_ENV_KEYS = [
18 'OPENAI_API_KEY',
19 'ANTHROPIC_API_KEY',
20 'DEEPINFRA_API_KEY',
21 'OPENROUTER_API_KEY',
22 'OPENROUTER_CHAT_MODEL',
23 'OPENROUTER_SITE_URL',
24 'OPENROUTER_APP_TITLE',
25 'KNOWTATION_CHAT_PROVIDER',
26 'KNOWTATION_CHAT_PREFER_ANTHROPIC',
27 ];
28
29 function clearChatEnv() {
30 for (const k of CHAT_ENV_KEYS) delete process.env[k];
31 }
32
33 function restoreEnv() {
34 for (const k of CHAT_ENV_KEYS) {
35 if (ORIG[k] === undefined) delete process.env[k];
36 else process.env[k] = ORIG[k];
37 }
38 }
39
40 describe('OpenRouter lane — performance', () => {
41 beforeEach(() => {
42 clearChatEnv();
43 process.env.KNOWTATION_CHAT_PROVIDER = 'openrouter';
44 process.env.OPENROUTER_API_KEY = 'or-perf';
45 globalThis.fetch = async () => ({
46 ok: true,
47 json: async () => ({ choices: [{ message: { content: 'fast' } }] }),
48 });
49 });
50
51 afterEach(() => {
52 globalThis.fetch = origFetch;
53 restoreEnv();
54 });
55
56 it('1000 sequential calls complete well under the regression ceiling', async () => {
57 const N = 1000;
58 const start = performance.now();
59 for (let i = 0; i < N; i += 1) {
60 await completeChat({}, { system: 's', user: `u-${i}` });
61 }
62 const elapsed = performance.now() - start;
63 // Generous ceiling: pure overhead should be a fraction of this on any machine.
64 assert.ok(elapsed < 5000, `1000 calls took ${elapsed.toFixed(1)}ms (ceiling 5000ms)`);
65 });
66
67 it('per-call overhead stays low even with a large prompt body', async () => {
68 const big = 'y'.repeat(50_000);
69 const N = 200;
70 const start = performance.now();
71 for (let i = 0; i < N; i += 1) {
72 await completeChat({}, { system: 's', user: big });
73 }
74 const avg = (performance.now() - start) / N;
75 assert.ok(avg < 25, `avg per-call overhead ${avg.toFixed(3)}ms (ceiling 25ms)`);
76 });
77
78 it('yields to the event loop (a pending timer fires around the call)', async () => {
79 let tick = false;
80 setTimeout(() => {
81 tick = true;
82 }, 0);
83 await completeChat({}, { system: 's', user: 'u' });
84 // Wait one real timer interval: if the lane had starved the loop, the 0ms timer
85 // would still be pending here. A deterministic 5ms wait guarantees the timers phase ran.
86 await new Promise((r) => setTimeout(r, 5));
87 assert.strictEqual(tick, true);
88 });
89 });
File History 2 commits
sha256:65ccb454656ea5acdea0a10e559b78bcde1eb6ff753ecc2911bc99d1c3d7cadd feat(calendar): enforce agent context tiers in retrieval AP… Human minor ⚠ 1 day ago
sha256:9103f98c89257ed2b01c237cea895dabb3e85ea337dccb1161c175e4422355b6 docs: accept Calendar Events v0 spec with Phase 0 security … Human 1 day ago