aaronrene / knowtation public
llm-complete-deepinfra.test.mjs
353 lines 12.1 KB
Raw
sha256:65ccb454656ea5acdea0a10e559b78bcde1eb6ff753ecc2911bc99d1c3d7cadd feat(calendar): enforce agent context tiers in retrieval AP… Human minor ⚠ breaking 1 day ago
1 /**
2 * DeepInfra provider routing for completeChat.
3 *
4 * Backward-compatibility contract: setting DEEPINFRA_API_KEY alongside an existing
5 * OPENAI_API_KEY must NOT change provider selection unless KNOWTATION_CHAT_PROVIDER=deepinfra
6 * is also set. This test guards against regressions for hosted Hub deploys that
7 * acquire a DeepInfra key for OpenClaw orchestration but keep OpenAI as primary chat.
8 */
9 import { describe, it, beforeEach, afterEach } from 'node:test';
10 import assert from 'node:assert';
11 import { completeChat } from '../lib/llm-complete.mjs';
12
13 const origFetch = globalThis.fetch;
14 const origOpenai = process.env.OPENAI_API_KEY;
15 const origAnthropic = process.env.ANTHROPIC_API_KEY;
16 const origDeepinfra = process.env.DEEPINFRA_API_KEY;
17 const origPrefer = process.env.KNOWTATION_CHAT_PREFER_ANTHROPIC;
18 const origProvider = process.env.KNOWTATION_CHAT_PROVIDER;
19
20 function setOrDelete(name, value) {
21 if (value === undefined) {
22 delete process.env[name];
23 } else {
24 process.env[name] = value;
25 }
26 }
27
28 function restoreEnv() {
29 setOrDelete('OPENAI_API_KEY', origOpenai);
30 setOrDelete('ANTHROPIC_API_KEY', origAnthropic);
31 setOrDelete('DEEPINFRA_API_KEY', origDeepinfra);
32 setOrDelete('KNOWTATION_CHAT_PREFER_ANTHROPIC', origPrefer);
33 setOrDelete('KNOWTATION_CHAT_PROVIDER', origProvider);
34 }
35
36 function clearChatEnv() {
37 delete process.env.OPENAI_API_KEY;
38 delete process.env.ANTHROPIC_API_KEY;
39 delete process.env.DEEPINFRA_API_KEY;
40 delete process.env.KNOWTATION_CHAT_PREFER_ANTHROPIC;
41 delete process.env.KNOWTATION_CHAT_PROVIDER;
42 }
43
44 describe('completeChat KNOWTATION_CHAT_PROVIDER=deepinfra', () => {
45 beforeEach(() => {
46 clearChatEnv();
47 });
48
49 afterEach(() => {
50 globalThis.fetch = origFetch;
51 restoreEnv();
52 });
53
54 it('explicit deepinfra: routes to DeepInfra even when OpenAI key is set', async () => {
55 process.env.KNOWTATION_CHAT_PROVIDER = 'deepinfra';
56 process.env.DEEPINFRA_API_KEY = 'di-test';
57 process.env.OPENAI_API_KEY = 'sk-openai-test';
58 const calls = [];
59 globalThis.fetch = async (url) => {
60 const u = String(url);
61 calls.push(u);
62 if (u.includes('api.deepinfra.com')) {
63 return {
64 ok: true,
65 json: async () => ({
66 choices: [{ message: { content: 'from-deepinfra' } }],
67 }),
68 };
69 }
70 return { ok: false, text: async () => 'should not reach openai' };
71 };
72 const out = await completeChat({}, { system: 's', user: 'u' });
73 assert.strictEqual(out, 'from-deepinfra');
74 assert.ok(calls.some((u) => u.includes('deepinfra.com')));
75 assert.ok(!calls.some((u) => u.includes('openai.com')));
76 });
77
78 it('explicit deepinfra without DEEPINFRA_API_KEY: throws actionable error', async () => {
79 process.env.KNOWTATION_CHAT_PROVIDER = 'deepinfra';
80 process.env.OPENAI_API_KEY = 'sk-openai-test';
81 globalThis.fetch = async () => {
82 throw new Error('fetch should not be called');
83 };
84 await assert.rejects(
85 () => completeChat({}, { system: 's', user: 'u' }),
86 /DEEPINFRA_API_KEY is not set/,
87 );
88 });
89
90 it('explicit deepinfra: falls back to OpenAI when DeepInfra returns 5xx', async () => {
91 process.env.KNOWTATION_CHAT_PROVIDER = 'deepinfra';
92 process.env.DEEPINFRA_API_KEY = 'di-test';
93 process.env.OPENAI_API_KEY = 'sk-openai-test';
94 let deepinfraCalls = 0;
95 let openaiCalls = 0;
96 globalThis.fetch = async (url) => {
97 const u = String(url);
98 if (u.includes('api.deepinfra.com')) {
99 deepinfraCalls++;
100 return { ok: false, status: 502, text: async () => 'bad gateway' };
101 }
102 if (u.includes('api.openai.com')) {
103 openaiCalls++;
104 return {
105 ok: true,
106 json: async () => ({
107 choices: [{ message: { content: 'from-openai-fallback' } }],
108 }),
109 };
110 }
111 return { ok: false, text: async () => 'unexpected' };
112 };
113 const out = await completeChat({}, { system: 's', user: 'u' });
114 assert.strictEqual(out, 'from-openai-fallback');
115 assert.strictEqual(deepinfraCalls, 1);
116 assert.strictEqual(openaiCalls, 1);
117 });
118
119 it('explicit deepinfra: falls back to Anthropic when DeepInfra fails and only Anthropic key is set', async () => {
120 process.env.KNOWTATION_CHAT_PROVIDER = 'deepinfra';
121 process.env.DEEPINFRA_API_KEY = 'di-test';
122 process.env.ANTHROPIC_API_KEY = 'sk-ant-test';
123 globalThis.fetch = async (url) => {
124 const u = String(url);
125 if (u.includes('api.deepinfra.com')) {
126 return { ok: false, status: 503, text: async () => 'unavailable' };
127 }
128 if (u.includes('api.anthropic.com')) {
129 return {
130 ok: true,
131 json: async () => ({
132 content: [{ text: 'from-claude-fallback' }],
133 }),
134 };
135 }
136 return { ok: false, text: async () => 'unexpected' };
137 };
138 const out = await completeChat({}, { system: 's', user: 'u' });
139 assert.strictEqual(out, 'from-claude-fallback');
140 });
141
142 it('explicit deepinfra: surfaces all provider errors when every fallback fails', async () => {
143 process.env.KNOWTATION_CHAT_PROVIDER = 'deepinfra';
144 process.env.DEEPINFRA_API_KEY = 'di-test';
145 process.env.OPENAI_API_KEY = 'sk-openai-test';
146 process.env.ANTHROPIC_API_KEY = 'sk-ant-test';
147 globalThis.fetch = async () => ({
148 ok: false,
149 status: 500,
150 text: async () => 'all-down',
151 });
152 await assert.rejects(
153 () => completeChat({}, { system: 's', user: 'u' }),
154 /DeepInfra chat failed.*OpenAI fallback failed.*Anthropic fallback failed/s,
155 );
156 });
157 });
158
159 describe('completeChat implicit DeepInfra (backward compatibility)', () => {
160 beforeEach(() => {
161 clearChatEnv();
162 });
163
164 afterEach(() => {
165 globalThis.fetch = origFetch;
166 restoreEnv();
167 });
168
169 it('only DEEPINFRA_API_KEY set: routes to DeepInfra', async () => {
170 process.env.DEEPINFRA_API_KEY = 'di-test';
171 const calls = [];
172 globalThis.fetch = async (url) => {
173 const u = String(url);
174 calls.push(u);
175 if (u.includes('api.deepinfra.com')) {
176 return {
177 ok: true,
178 json: async () => ({
179 choices: [{ message: { content: 'from-deepinfra-implicit' } }],
180 }),
181 };
182 }
183 return { ok: false, text: async () => 'should not reach' };
184 };
185 const out = await completeChat({}, { system: 's', user: 'u' });
186 assert.strictEqual(out, 'from-deepinfra-implicit');
187 assert.ok(calls.length === 1 && calls[0].includes('deepinfra.com'));
188 });
189
190 it('DEEPINFRA + OPENAI both set, no explicit provider: keeps OpenAI as default (no regression)', async () => {
191 process.env.DEEPINFRA_API_KEY = 'di-test';
192 process.env.OPENAI_API_KEY = 'sk-openai-test';
193 const calls = [];
194 globalThis.fetch = async (url) => {
195 const u = String(url);
196 calls.push(u);
197 if (u.includes('api.openai.com')) {
198 return {
199 ok: true,
200 json: async () => ({
201 choices: [{ message: { content: 'from-openai-default' } }],
202 }),
203 };
204 }
205 return { ok: false, text: async () => 'unexpected provider' };
206 };
207 const out = await completeChat({}, { system: 's', user: 'u' });
208 assert.strictEqual(out, 'from-openai-default');
209 assert.ok(calls.every((u) => !u.includes('deepinfra.com')));
210 });
211
212 it('DEEPINFRA + ANTHROPIC both set, no explicit provider: keeps Anthropic as default (no regression)', async () => {
213 process.env.DEEPINFRA_API_KEY = 'di-test';
214 process.env.ANTHROPIC_API_KEY = 'sk-ant-test';
215 const calls = [];
216 globalThis.fetch = async (url) => {
217 const u = String(url);
218 calls.push(u);
219 if (u.includes('api.anthropic.com')) {
220 return {
221 ok: true,
222 json: async () => ({
223 content: [{ text: 'from-claude-default' }],
224 }),
225 };
226 }
227 return { ok: false, text: async () => 'unexpected' };
228 };
229 const out = await completeChat({}, { system: 's', user: 'u' });
230 assert.strictEqual(out, 'from-claude-default');
231 assert.ok(calls.every((u) => !u.includes('deepinfra.com')));
232 });
233 });
234
235 describe('completeChat KNOWTATION_CHAT_PROVIDER=openai|anthropic explicit lock', () => {
236 beforeEach(() => {
237 clearChatEnv();
238 });
239
240 afterEach(() => {
241 globalThis.fetch = origFetch;
242 restoreEnv();
243 });
244
245 it('explicit openai: uses OpenAI even when DEEPINFRA_API_KEY is set', async () => {
246 process.env.KNOWTATION_CHAT_PROVIDER = 'openai';
247 process.env.OPENAI_API_KEY = 'sk-openai-test';
248 process.env.DEEPINFRA_API_KEY = 'di-test';
249 globalThis.fetch = async (url) => {
250 const u = String(url);
251 if (u.includes('api.openai.com')) {
252 return {
253 ok: true,
254 json: async () => ({ choices: [{ message: { content: 'from-openai-locked' } }] }),
255 };
256 }
257 return { ok: false, text: async () => 'should not reach' };
258 };
259 const out = await completeChat({}, { system: 's', user: 'u' });
260 assert.strictEqual(out, 'from-openai-locked');
261 });
262
263 it('explicit anthropic: uses Anthropic even when DEEPINFRA_API_KEY and OPENAI_API_KEY are set', async () => {
264 process.env.KNOWTATION_CHAT_PROVIDER = 'anthropic';
265 process.env.ANTHROPIC_API_KEY = 'sk-ant-test';
266 process.env.OPENAI_API_KEY = 'sk-openai-test';
267 process.env.DEEPINFRA_API_KEY = 'di-test';
268 globalThis.fetch = async (url) => {
269 const u = String(url);
270 if (u.includes('api.anthropic.com')) {
271 return {
272 ok: true,
273 json: async () => ({ content: [{ text: 'from-claude-locked' }] }),
274 };
275 }
276 return { ok: false, text: async () => 'should not reach' };
277 };
278 const out = await completeChat({}, { system: 's', user: 'u' });
279 assert.strictEqual(out, 'from-claude-locked');
280 });
281
282 it('explicit openai without OPENAI_API_KEY: throws actionable error', async () => {
283 process.env.KNOWTATION_CHAT_PROVIDER = 'openai';
284 process.env.DEEPINFRA_API_KEY = 'di-test';
285 globalThis.fetch = async () => ({ ok: true, json: async () => ({}) });
286 await assert.rejects(
287 () => completeChat({}, { system: 's', user: 'u' }),
288 /OPENAI_API_KEY is not set/,
289 );
290 });
291 });
292
293 describe('completeChat DEEPINFRA_CHAT_MODEL override', () => {
294 beforeEach(() => {
295 clearChatEnv();
296 });
297
298 afterEach(() => {
299 globalThis.fetch = origFetch;
300 restoreEnv();
301 delete process.env.DEEPINFRA_CHAT_MODEL;
302 });
303
304 it('uses default Qwen/Qwen2.5-72B-Instruct when DEEPINFRA_CHAT_MODEL is unset', async () => {
305 process.env.KNOWTATION_CHAT_PROVIDER = 'deepinfra';
306 process.env.DEEPINFRA_API_KEY = 'di-test';
307 let observedModel;
308 globalThis.fetch = async (url, init) => {
309 observedModel = JSON.parse(init.body).model;
310 return {
311 ok: true,
312 json: async () => ({ choices: [{ message: { content: 'ok' } }] }),
313 };
314 };
315 await completeChat({}, { system: 's', user: 'u' });
316 assert.strictEqual(observedModel, 'Qwen/Qwen2.5-72B-Instruct');
317 });
318
319 it('honors DEEPINFRA_CHAT_MODEL env override (e.g. cheap 8B for review hints)', async () => {
320 process.env.KNOWTATION_CHAT_PROVIDER = 'deepinfra';
321 process.env.DEEPINFRA_API_KEY = 'di-test';
322 process.env.DEEPINFRA_CHAT_MODEL = 'meta-llama/Meta-Llama-3.1-8B-Instruct';
323 let observedModel;
324 globalThis.fetch = async (url, init) => {
325 observedModel = JSON.parse(init.body).model;
326 return {
327 ok: true,
328 json: async () => ({ choices: [{ message: { content: 'ok' } }] }),
329 };
330 };
331 await completeChat({}, { system: 's', user: 'u' });
332 assert.strictEqual(observedModel, 'meta-llama/Meta-Llama-3.1-8B-Instruct');
333 });
334
335 it('honors config.llm.deepinfra_chat_model over env (caller-side override)', async () => {
336 process.env.KNOWTATION_CHAT_PROVIDER = 'deepinfra';
337 process.env.DEEPINFRA_API_KEY = 'di-test';
338 process.env.DEEPINFRA_CHAT_MODEL = 'meta-llama/Meta-Llama-3.1-8B-Instruct';
339 let observedModel;
340 globalThis.fetch = async (url, init) => {
341 observedModel = JSON.parse(init.body).model;
342 return {
343 ok: true,
344 json: async () => ({ choices: [{ message: { content: 'ok' } }] }),
345 };
346 };
347 await completeChat(
348 { llm: { deepinfra_chat_model: 'mistralai/Mixtral-8x7B-Instruct-v0.1' } },
349 { system: 's', user: 'u' },
350 );
351 assert.strictEqual(observedModel, 'mistralai/Mixtral-8x7B-Instruct-v0.1');
352 });
353 });
File History 2 commits
sha256:65ccb454656ea5acdea0a10e559b78bcde1eb6ff753ecc2911bc99d1c3d7cadd feat(calendar): enforce agent context tiers in retrieval AP… Human minor ⚠ 1 day ago
sha256:9103f98c89257ed2b01c237cea895dabb3e85ea337dccb1161c175e4422355b6 docs: accept Calendar Events v0 spec with Phase 0 security … Human 1 day ago