aaronrene / knowtation public
notion-hub-connector.mjs
367 lines 13.8 KB
Raw
sha256:49f768e5fb72e8d17321410817422fc5cab8f0a1b66a1e1da8bdd4cd251c4f7e Merge 'feat/ourware-landing-rebrand' into 'main' — proposal… Human 18 days ago
1 /**
2 * Inert process-wide Hub-key Notion connector.
3 *
4 * There is no OAuth flow and connector records never contain NOTION_API_KEY.
5 */
6
7 import { fetchNotionPageMarkdown } from '../importers/notion.mjs';
8 import {
9 connectorForClient,
10 getConnector,
11 listConnectors,
12 newConnectorId,
13 saveConnector,
14 } from './docs-connector-store.mjs';
15 import { proposeDocsImports } from './docs-import-propose.mjs';
16
17 export const DOCS_NOTION_HUB_KEY_AUTHORIZED = false;
18 export const NOTION_PAGE_ID_RE = /^(?:[A-Fa-f0-9]{32}|[A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12})$/;
19 const NOTION_API_BASE = 'https://api.notion.com/v1';
20 const NOTION_VERSION = '2022-06-28';
21 const activeSyncs = new Set();
22
23 export function isDocsNotionHubKeyEnabled({ authorizedOverride } = {}) {
24 if (authorizedOverride === true) return true;
25 if (authorizedOverride === false) return false;
26 return DOCS_NOTION_HUB_KEY_AUTHORIZED === true;
27 }
28
29 function result(status, code) {
30 return { ok: false, status, code };
31 }
32
33 function notAuthorized() {
34 return result(501, 'NOT_AUTHORIZED');
35 }
36
37 async function sleep(ms) {
38 if (ms > 0) await new Promise((resolve) => setTimeout(resolve, ms));
39 }
40
41 async function searchWithBackoff(ctx, client, params) {
42 for (let attempt = 0; attempt < 3; attempt++) {
43 const response = await client.search(params);
44 if (response?.status !== 429) return response;
45 if (attempt < 2) {
46 const wait = Number.isFinite(response.retryAfterMs)
47 ? Math.max(0, Math.min(response.retryAfterMs, 60_000))
48 : 0;
49 await (ctx.sleepFn ?? sleep)(wait);
50 }
51 }
52 return { status: 429 };
53 }
54
55 function apiKeyFrom(env = process.env) {
56 return typeof env?.NOTION_API_KEY === 'string' ? env.NOTION_API_KEY.trim() : '';
57 }
58
59 function titleFromPage(page) {
60 const properties = page?.properties && typeof page.properties === 'object' ? page.properties : {};
61 for (const property of Object.values(properties)) {
62 if (property?.type !== 'title' || !Array.isArray(property.title)) continue;
63 const title = property.title.map((part) => part?.plain_text ?? '').join('').trim();
64 if (title) return title.slice(0, 512);
65 }
66 return 'Untitled Notion page';
67 }
68
69 function normalizeNotionResult(row) {
70 return {
71 file_id: typeof row?.id === 'string' ? row.id : '',
72 name: row?.object === 'page' ? titleFromPage(row) : 'Notion database',
73 mime: row?.object === 'page' ? 'application/vnd.notion.page' : 'application/vnd.notion.database',
74 modified: typeof row?.last_edited_time === 'string' ? row.last_edited_time : null,
75 size: 0,
76 importable: row?.object === 'page',
77 };
78 }
79
80 export function handleBeginNotionConnector(ctx) {
81 if (!isDocsNotionHubKeyEnabled({ authorizedOverride: ctx.authorizedOverride })) return notAuthorized();
82 const body = ctx.body && typeof ctx.body === 'object' && !Array.isArray(ctx.body) ? ctx.body : null;
83 if (!body || Object.keys(body).some((key) => !['provider', 'display_name', 'return_url'].includes(key))) {
84 return result(400, 'BAD_REQUEST');
85 }
86 if (body.provider !== 'notion') return result(400, 'PROVIDER_DENIED');
87 const connector = {
88 connector_id: newConnectorId(),
89 provider: 'notion',
90 display_name: typeof body.display_name === 'string' && body.display_name.trim()
91 ? body.display_name.trim().slice(0, 128)
92 : 'Notion',
93 status: apiKeyFrom(ctx.env) ? 'connected' : 'needs_reauth',
94 account_sub: null,
95 oauth_ref: null,
96 sync_cursor: null,
97 last_sync_at: null,
98 last_sync_error: 'none',
99 file_count: 0,
100 revoked_at: null,
101 oauth_pending: null,
102 };
103 saveConnector(ctx.dataDir, ctx.vaultId, connector);
104 return {
105 ok: true,
106 status: 200,
107 payload: { connector_id: connector.connector_id, status: connector.status },
108 };
109 }
110
111 export function handleListNotionConnectors(ctx) {
112 if (!isDocsNotionHubKeyEnabled({ authorizedOverride: ctx.authorizedOverride })) return notAuthorized();
113 return {
114 ok: true,
115 status: 200,
116 payload: {
117 schema: 'knowtation.docs_connectors/v0',
118 connectors: listConnectors(ctx.dataDir, ctx.vaultId)
119 .filter((connector) => connector.provider === 'notion')
120 .map(connectorForClient),
121 },
122 };
123 }
124
125 function connectedNotion(ctx) {
126 const connector = getConnector(ctx.dataDir, ctx.vaultId, ctx.connectorId);
127 if (!connector || connector.status === 'revoked') return { response: result(404, 'CONNECTOR_NOT_FOUND') };
128 if (connector.provider !== 'notion') return { response: result(400, 'PROVIDER_DENIED') };
129 const key = apiKeyFrom(ctx.env);
130 if (!key || connector.status === 'needs_reauth') {
131 if (connector.status !== 'needs_reauth') {
132 connector.status = 'needs_reauth';
133 connector.last_sync_error = 'auth_expired';
134 saveConnector(ctx.dataDir, ctx.vaultId, connector);
135 }
136 return { response: result(409, 'NEEDS_REAUTH') };
137 }
138 if (connector.status !== 'connected') return { response: result(400, 'BAD_REQUEST') };
139 return { connector, apiKey: key };
140 }
141
142 export function createProductionNotionClient({ fetchImpl = globalThis.fetch } = {}) {
143 if (typeof fetchImpl !== 'function') throw new TypeError('fetch implementation is required');
144 const headers = (apiKey) => ({
145 Authorization: `Bearer ${apiKey}`,
146 'Notion-Version': NOTION_VERSION,
147 'Content-Type': 'application/json',
148 Accept: 'application/json',
149 });
150 return {
151 search: async ({ apiKey, startCursor }) => {
152 const response = await fetchImpl(`${NOTION_API_BASE}/search`, {
153 method: 'POST',
154 headers: headers(apiKey),
155 body: JSON.stringify({
156 page_size: 50,
157 ...(startCursor ? { start_cursor: startCursor } : {}),
158 sort: { direction: 'descending', timestamp: 'last_edited_time' },
159 }),
160 });
161 const body = await response.json();
162 const retryAfter = Number.parseFloat(response.headers.get('retry-after') ?? '');
163 return {
164 ...body,
165 status: response.status,
166 ...(Number.isFinite(retryAfter) ? { retryAfterMs: retryAfter * 1000 } : {}),
167 };
168 },
169 fetchPageMarkdown: ({ pageId, apiKey }) => fetchNotionPageMarkdown(pageId, { apiKey, fetchImpl }),
170 };
171 }
172
173 export function createFakeNotionClient(fixtures = {}) {
174 return {
175 search: async (args) => typeof fixtures.search === 'function'
176 ? fixtures.search(args)
177 : { results: fixtures.results ?? [], next_cursor: fixtures.next_cursor ?? null, has_more: false, status: 200 },
178 fetchPageMarkdown: async (args) => typeof fixtures.fetchPageMarkdown === 'function'
179 ? fixtures.fetchPageMarkdown(args)
180 : fixtures.markdownByPage?.[args.pageId] ?? '',
181 };
182 }
183
184 function notionClient(ctx) {
185 return ctx.notionClient ?? createProductionNotionClient({ fetchImpl: ctx.fetchImpl });
186 }
187
188 export async function handleListNotionConnectorFiles(ctx) {
189 if (!isDocsNotionHubKeyEnabled({ authorizedOverride: ctx.authorizedOverride })) return notAuthorized();
190 const found = connectedNotion(ctx);
191 if (found.response) return found.response;
192 const pageToken = ctx.query?.page_token;
193 if (pageToken !== undefined && (typeof pageToken !== 'string' || pageToken.length > 2048)) {
194 return result(400, 'BAD_REQUEST');
195 }
196 const client = notionClient(ctx);
197 const response = await searchWithBackoff(
198 ctx,
199 client,
200 { apiKey: found.apiKey, ...(pageToken ? { startCursor: pageToken } : {}) },
201 );
202 if (response.status === 429) return result(429, 'RATE_LIMITED');
203 if (response.status && response.status >= 400) return result(502, 'PROVIDER_ERROR');
204 const files = (Array.isArray(response.results) ? response.results : []).slice(0, 50).map(normalizeNotionResult);
205 found.connector.file_count = files.length;
206 found.connector.last_sync_error = 'none';
207 saveConnector(ctx.dataDir, ctx.vaultId, found.connector);
208 return {
209 ok: true,
210 status: 200,
211 payload: {
212 files,
213 ...(typeof response.next_cursor === 'string' && response.next_cursor
214 ? { next_page_token: response.next_cursor }
215 : {}),
216 },
217 };
218 }
219
220 async function fetchNotionItems(ctx, found, pageIds) {
221 const client = notionClient(ctx);
222 const items = [];
223 const skips = [];
224 let batchBytes = 0;
225 for (const pageId of pageIds) {
226 let markdown;
227 try {
228 markdown = await client.fetchPageMarkdown({ pageId, apiKey: found.apiKey });
229 } catch {
230 skips.push({ source_id: pageId, reason: 'not_found' });
231 continue;
232 }
233 const size = Buffer.byteLength(typeof markdown === 'string' ? markdown : '', 'utf8');
234 if (size > 25_000_000) {
235 skips.push({ source_id: pageId, reason: 'too_large' });
236 continue;
237 }
238 if (!markdown.trim()) {
239 skips.push({ source_id: pageId, reason: 'empty_extract' });
240 continue;
241 }
242 batchBytes += size;
243 if (batchBytes > 80_000_000) throw Object.assign(new TypeError('import batch exceeds byte cap'), { code: 'BAD_REQUEST' });
244 items.push({ source_id: pageId, name: pageId, markdown, size });
245 }
246 return { items, skips };
247 }
248
249 export async function handleImportNotionConnectorFiles(ctx) {
250 if (!isDocsNotionHubKeyEnabled({ authorizedOverride: ctx.authorizedOverride })) return notAuthorized();
251 const found = connectedNotion(ctx);
252 if (found.response) return found.response;
253 const body = ctx.body && typeof ctx.body === 'object' && !Array.isArray(ctx.body) ? ctx.body : null;
254 if (!body || Object.keys(body).some((key) => key !== 'file_ids')) return result(400, 'BAD_REQUEST');
255 const pageIds = body.file_ids;
256 if (!Array.isArray(pageIds) || pageIds.length < 1 || pageIds.length > 20 || !pageIds.every((id) => NOTION_PAGE_ID_RE.test(id))) {
257 return result(400, 'BAD_REQUEST');
258 }
259 let fetched;
260 try {
261 fetched = await fetchNotionItems(ctx, found, pageIds);
262 } catch (error) {
263 return error?.code === 'BAD_REQUEST' ? result(400, 'BAD_REQUEST') : result(502, 'PROVIDER_ERROR');
264 }
265 const proposed = fetched.items.length
266 ? proposeDocsImports({
267 dataDir: ctx.dataDir,
268 vaultPath: ctx.vaultPath,
269 vaultId: ctx.vaultId,
270 connectorId: ctx.connectorId,
271 provider: 'notion',
272 items: fetched.items,
273 now: ctx.now,
274 createProposalFn: ctx.createProposalFn,
275 loadProposalsFn: ctx.loadProposalsFn,
276 listMarkdownFilesFn: ctx.listMarkdownFilesFn,
277 readNoteFn: ctx.readNoteFn,
278 })
279 : { proposed: 0, skipped: 0, proposal_ids: [], skip_details: [] };
280 const skipDetails = [...fetched.skips, ...proposed.skip_details];
281 return {
282 ok: true,
283 status: 200,
284 payload: {
285 proposed: proposed.proposed,
286 skipped: skipDetails.length,
287 proposal_ids: proposed.proposal_ids,
288 skip_details: skipDetails,
289 },
290 };
291 }
292
293 export async function handleSyncNotionConnector(ctx) {
294 if (!isDocsNotionHubKeyEnabled({ authorizedOverride: ctx.authorizedOverride })) return notAuthorized();
295 const found = connectedNotion(ctx);
296 if (found.response) return found.response;
297 const now = ctx.now ?? Date.now();
298 const last = found.connector.last_sync_at ? Date.parse(found.connector.last_sync_at) : 0;
299 if (activeSyncs.has(ctx.connectorId) || (Number.isFinite(last) && now - last < 60_000)) {
300 return result(429, 'RATE_LIMITED');
301 }
302 activeSyncs.add(ctx.connectorId);
303 try {
304 const client = notionClient(ctx);
305 const response = await searchWithBackoff(ctx, client, {
306 apiKey: found.apiKey,
307 ...(found.connector.sync_cursor ? { startCursor: found.connector.sync_cursor } : {}),
308 });
309 if (response.status === 429) return result(429, 'RATE_LIMITED');
310 if (response.status && response.status >= 400) return result(502, 'PROVIDER_ERROR');
311 const pages = (response.results ?? []).filter((row) => row?.object === 'page').slice(0, 20);
312 const ids = pages.map((row) => row.id).filter((id) => NOTION_PAGE_ID_RE.test(id));
313 const fetched = await fetchNotionItems(ctx, found, ids);
314 const proposed = fetched.items.length
315 ? proposeDocsImports({
316 dataDir: ctx.dataDir,
317 vaultPath: ctx.vaultPath,
318 vaultId: ctx.vaultId,
319 connectorId: ctx.connectorId,
320 provider: 'notion',
321 items: fetched.items,
322 now,
323 createProposalFn: ctx.createProposalFn,
324 loadProposalsFn: ctx.loadProposalsFn,
325 listMarkdownFilesFn: ctx.listMarkdownFilesFn,
326 readNoteFn: ctx.readNoteFn,
327 })
328 : { proposed: 0, skipped: 0 };
329 found.connector.sync_cursor = typeof response.next_cursor === 'string' ? response.next_cursor : null;
330 found.connector.last_sync_at = new Date(now).toISOString();
331 found.connector.last_sync_error = 'none';
332 found.connector.file_count = pages.length;
333 saveConnector(ctx.dataDir, ctx.vaultId, found.connector);
334 return {
335 ok: true,
336 status: 200,
337 payload: {
338 proposed: proposed.proposed,
339 skipped: fetched.skips.length + proposed.skipped,
340 last_sync_at: found.connector.last_sync_at,
341 },
342 };
343 } catch {
344 found.connector.last_sync_error = 'network_error';
345 saveConnector(ctx.dataDir, ctx.vaultId, found.connector);
346 return result(502, 'PROVIDER_ERROR');
347 } finally {
348 activeSyncs.delete(ctx.connectorId);
349 }
350 }
351
352 export function handleRevokeNotionConnector(ctx) {
353 if (!isDocsNotionHubKeyEnabled({ authorizedOverride: ctx.authorizedOverride })) return notAuthorized();
354 const connector = getConnector(ctx.dataDir, ctx.vaultId, ctx.connectorId);
355 if (!connector || connector.status === 'revoked') return result(404, 'CONNECTOR_NOT_FOUND');
356 if (connector.provider !== 'notion') return result(400, 'PROVIDER_DENIED');
357 connector.status = 'revoked';
358 connector.revoked_at = new Date(ctx.now ?? Date.now()).toISOString();
359 connector.sync_cursor = null;
360 connector.oauth_ref = null;
361 connector.oauth_pending = null;
362 saveConnector(ctx.dataDir, ctx.vaultId, connector);
363 return { ok: true, status: 200, payload: { revoked: true } };
364 }
365
366 export const handleListNotionConnectorPages = handleListNotionConnectorFiles;
367 export const handleImportNotionConnectorPages = handleImportNotionConnectorFiles;
File History 1 commit
sha256:49f768e5fb72e8d17321410817422fc5cab8f0a1b66a1e1da8bdd4cd251c4f7e Merge 'feat/ourware-landing-rebrand' into 'main' — proposal… Human 18 days ago