diff --git a/scripts/render-check.mjs b/scripts/render-check.mjs index e32508e..d5a4e6e 100644 --- a/scripts/render-check.mjs +++ b/scripts/render-check.mjs @@ -23,16 +23,20 @@ const PORT = Number(process.env.PORT || 8799); const RUNS = Number(process.env.RUNS || 2); const DB_VIEWS = ["table", "board", "gallery", "list", "calendar", "timeline"]; +const CRAWLER = "Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)"; const TARGETS = { worker: { origin: `https://${DOMAIN}`, + // For requests outside the browser, which bypass its routing to wrangler + direct: `https://127.0.0.1:${PORT}`, root: "/", isRoot: (p) => p === "/", isSubpage: (p) => p === "/subpage", }, notion: { origin: NOTION_SITE, + direct: NOTION_SITE, root: `/${ROOT_PAGE}`, isRoot: (p) => p.endsWith(ROOT_PAGE), isSubpage: (p) => p.endsWith(SUBPAGE), @@ -233,6 +237,19 @@ const CHECKS = { s = await state(page); return target.isSubpage(s.path) && s.h1 === "Render Check Subpage" ? null : `after forward: ${s.path} "${s.h1}"`; }, + // Crawlers get the server-rendered page, with the canonical URL and links on this site + "crawler content": async ({ context, target }) => { + const response = await context.request.get(target.direct + target.root, { headers: { "user-agent": CRAWLER } }); + const html = await response.text(); + if (!html.includes("Test page for Worknot")) return "page text missing"; + const host = new URL(target.origin).host; + const canonical = [...html.matchAll(/ new URL(m[1]).host); + if (canonical.length !== 1 || canonical[0] !== host) return `canonical: ${canonical.join(", ") || "none"}`; + const offsite = [...html.matchAll(/]*href="(https:\/\/[^"]+\.notion\.site\/[^"]*)"/g)] + .map((m) => m[1]) + .find((href) => new URL(href).host !== host); + return offsite ? `links to notion.site: ${offsite}` : null; + }, "no errors": async ({ errors, failures }) => { const problems = [...errors, ...failures]; return problems.length ? problems.slice(0, 5).join("; ") : null; @@ -240,7 +257,7 @@ const CHECKS = { }; const GROUPS = [ - ["page renders", "database views", "database rows and formula", "images load", "pdf and file blocks", "mermaid diagram", "embeds and bookmarks", "anchor link", "no errors"], + ["page renders", "database views", "database rows and formula", "images load", "pdf and file blocks", "mermaid diagram", "embeds and bookmarks", "anchor link", "crawler content", "no errors"], // Checks that navigate run on a page of their own ["database item stays on domain"], ["navigation and history"], diff --git a/src/code.ts b/src/code.ts index 9358013..14eec73 100644 --- a/src/code.ts +++ b/src/code.ts @@ -795,6 +795,24 @@ ${ 'accept-language': 'en-US,en;q=0.9' }; + // Notion answers bot-like user agents with a server-rendered page that has the + // content as plain HTML. The visitor's user agent is not forwarded, because + // Notion rejects Googlebot or Bingbot requests that come from a Worker. + const CRAWLER_FETCH_HEADERS = { + ...PAGE_FETCH_HEADERS, + 'user-agent': 'Mozilla/5.0 (compatible; WorknotBot/1.0; +https://worknot.classmethod.live/)' + }; + + // The server-rendered page has no Notion app, so people must never get it: + // "bot" also matches CUBOT phones, which are excluded. + const CRAWLER_USER_AGENT = /bot\\b|crawler|spider|chatgpt-user|claude-user|perplexity-user|meta-externalagent/i; + + function isCrawler(request) { + const userAgent = request.headers.get('User-Agent') || ''; + return (CRAWLER_USER_AGENT.test(userAgent) && !/cubot/i.test(userAgent)) + || (request.headers.get('Accept') || '').includes('text/markdown'); + } + // Forward Notion routing headers (e.g. x-notion-cell for fanout requests). // Without them Notion keeps answering with fanoutData and pages render as 404. function buildApiHeaders(request) { @@ -916,6 +934,7 @@ ${ return notFoundResponse(url, true); } let response; + const pageFetchHeaders = isCrawler(request) ? CRAWLER_FETCH_HEADERS : PAGE_FETCH_HEADERS; const matchedSlug = pathToSlug(url.pathname); const isAppJs = url.pathname.startsWith('/app') && url.pathname.endsWith('js'); const isAssetJs = url.pathname.startsWith('/_assets/') && url.pathname.endsWith('.js'); @@ -1009,7 +1028,7 @@ ${ const pageId = SLUG_TO_PAGE[matchedSlug]; url.pathname = '/' + pageId; response = await fetch(url.toString(), { - headers: PAGE_FETCH_HEADERS, + headers: pageFetchHeaders, }); response = new Response(response.body, response); response.headers.delete('Content-Security-Policy'); @@ -1017,7 +1036,7 @@ ${ return appendJavascript(response, SLUG_TO_PAGE, matchedSlug, url); } else { response = await fetch(url.toString(), { - headers: PAGE_FETCH_HEADERS, + headers: pageFetchHeaders, }); // HTMLRewriter parses any body regardless of Content-Type, so stream // non-HTML responses (CSS, images, files) through untouched @@ -1114,13 +1133,34 @@ ${ class LinkRewriter { element(element) { + const rel = element.getAttribute('rel'); + // The server-rendered page names its notion.site URL as canonical; HeadRewriter adds ours + if (rel === 'canonical') { + element.remove(); + return; + } // Remove Notion's default favicon links when custom favicon is set (Issue #16) - if (FAVICON_URL !== '') { - const rel = element.getAttribute('rel'); - if (rel && (rel.includes('icon') || rel === 'apple-touch-icon')) { - element.remove(); - } + if (FAVICON_URL !== '' && rel && (rel.includes('icon') || rel === 'apple-touch-icon')) { + element.remove(); + } + } + } + + // Links in the server-rendered page point to notion.site. Keep them on this + // domain, with the slug for mapped pages. + class AnchorRewriter { + element(element) { + let link; + try { + link = new URL(element.getAttribute('href')); + } catch (e) { + return; } + if (link.hostname !== NOTION_SITE_DOMAIN) return; + link.hostname = MY_DOMAIN; + const pageId = link.pathname.match(/[0-9a-f]{32}$/)?.[0]; + if (pageId && PAGE_TO_SLUG[pageId] !== undefined) link.pathname = '/' + PAGE_TO_SLUG[pageId]; + element.setAttribute('href', link.toString()); } } @@ -1587,6 +1627,7 @@ ${ .on('title', metaRewriter) .on('meta', metaRewriter) .on('link', linkRewriter) + .on('a[href]', new AnchorRewriter()) .on('head', headRewriter) .on('body', new BodyRewriter(SLUG_TO_PAGE, res.status === 404)) .transform(res); diff --git a/test/generated-worker.test.mjs b/test/generated-worker.test.mjs index 3edc286..158f6fd 100644 --- a/test/generated-worker.test.mjs +++ b/test/generated-worker.test.mjs @@ -136,6 +136,33 @@ test('malformed JSON passes through without fabricated site context', async (t) assert.equal(forwarded.body, '{invalid'); }); +// Returns the user agent the worker sends to Notion for a page request +async function pageFetchUserAgent(t, headers) { + globalThis.HTMLRewriter = class { on() { return this; } transform(response) { return response; } }; + t.after(() => { delete globalThis.HTMLRewriter; }); + const upstream = t.mock.method(globalThis, 'fetch', async () => new Response('', { + headers: { 'Content-Type': 'text/html' }, + })); + await run(worker, new Request('https://notes.example.com/chapter', { headers })); + return new Headers(upstream.mock.calls[0].arguments[1].headers).get('user-agent'); +} + +const WORKNOT_BOT = 'Mozilla/5.0 (compatible; WorknotBot/1.0; +https://worknot.classmethod.live/)'; +const CHROME = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36'; + +for (const [visitor, headers, expected] of [ + ['GPTBot', { 'User-Agent': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)' }, WORKNOT_BOT], + ['ChatGPT-User', { 'User-Agent': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot' }, WORKNOT_BOT], + ['Googlebot', { 'User-Agent': 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' }, WORKNOT_BOT], + ['an agent asking for Markdown', { 'User-Agent': 'curl/8.7.1', Accept: 'text/markdown' }, WORKNOT_BOT], + ['a browser', { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36' }, CHROME], + ['a CUBOT phone', { 'User-Agent': 'Mozilla/5.0 (Linux; Android 10; CUBOT X30) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Mobile Safari/537.36' }, CHROME], +]) { + test(`pages for ${visitor} are fetched as ${expected === WORKNOT_BOT ? 'a crawler' : 'a browser'}`, async (t) => { + assert.equal(await pageFetchUserAgent(t, headers), expected); + }); +} + test('cross-origin preflight requests get no CORS headers', async (t) => { const upstream = t.mock.method(globalThis, 'fetch', async () => new Response()); const response = await run(worker, new Request('https://notes.example.com/api/v3/getPublicPageData', {