Serve Notion's server-rendered pages to crawlers

scripts/render-check.mjs

@@ -23,16 +23,20 @@ const PORT = Number(process.env.PORT || 8799);
 const RUNS = Number(process.env.RUNS || 2);
 
 const DB_VIEWS = ["table", "board", "gallery", "list", "calendar", "timeline"];
+const CRAWLER = "Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)";
 
 const TARGETS = {
   worker: {
     origin: `https://${DOMAIN}`,
+    // For requests outside the browser, which bypass its routing to wrangler
+    direct: `https://127.0.0.1:${PORT}`,
     root: "/",
     isRoot: (p) => p === "/",
     isSubpage: (p) => p === "/subpage",
   },
   notion: {
     origin: NOTION_SITE,
+    direct: NOTION_SITE,
     root: `/${ROOT_PAGE}`,
     isRoot: (p) => p.endsWith(ROOT_PAGE),
     isSubpage: (p) => p.endsWith(SUBPAGE),
@@ -233,14 +237,27 @@ const CHECKS = {
     s = await state(page);
     return target.isSubpage(s.path) && s.h1 === "Render Check Subpage" ? null : `after forward: ${s.path} "${s.h1}"`;
   },
+  // Crawlers get the server-rendered page, with the canonical URL and links on this site
+  "crawler content": async ({ context, target }) => {
+    const response = await context.request.get(target.direct + target.root, { headers: { "user-agent": CRAWLER } });
+    const html = await response.text();
+    if (!html.includes("Test page for Worknot")) return "page text missing";
+    const host = new URL(target.origin).host;
+    const canonical = [...html.matchAll(/<link rel="canonical" href="([^"]+)"/g)].map((m) => new URL(m[1]).host);
+    if (canonical.length !== 1 || canonical[0] !== host) return `canonical: ${canonical.join(", ") || "none"}`;
+    const offsite = [...html.matchAll(/<a [^>]*href="(https:\/\/[^"]+\.notion\.site\/[^"]*)"/g)]
+      .map((m) => m[1])
+      .find((href) => new URL(href).host !== host);
+    return offsite ? `links to notion.site: ${offsite}` : null;
+  },
   "no errors": async ({ errors, failures }) => {
     const problems = [...errors, ...failures];
     return problems.length ? problems.slice(0, 5).join("; ") : null;
   },
 };
 
 const GROUPS = [
-  ["page renders", "database views", "database rows and formula", "images load", "pdf and file blocks", "mermaid diagram", "embeds and bookmarks", "anchor link", "no errors"],
+  ["page renders", "database views", "database rows and formula", "images load", "pdf and file blocks", "mermaid diagram", "embeds and bookmarks", "anchor link", "crawler content", "no errors"],
   // Checks that navigate run on a page of their own
   ["database item stays on domain"],
   ["navigation and history"],

src/code.ts

@@ -795,6 +795,24 @@ ${
     'accept-language': 'en-US,en;q=0.9'
   };
 
+  // Notion answers bot-like user agents with a server-rendered page that has the
+  // content as plain HTML. The visitor's user agent is not forwarded, because
+  // Notion rejects Googlebot or Bingbot requests that come from a Worker.
+  const CRAWLER_FETCH_HEADERS = {
+    ...PAGE_FETCH_HEADERS,
+    'user-agent': 'Mozilla/5.0 (compatible; WorknotBot/1.0; +https://worknot.classmethod.live/)'
+  };
+
+  // The server-rendered page has no Notion app, so people must never get it:
+  // "bot" also matches CUBOT phones, which are excluded.
+  const CRAWLER_USER_AGENT = /bot\\b|crawler|spider|chatgpt-user|claude-user|perplexity-user|meta-externalagent/i;
+
+  function isCrawler(request) {
+    const userAgent = request.headers.get('User-Agent') || '';
+    return (CRAWLER_USER_AGENT.test(userAgent) && !/cubot/i.test(userAgent))
+      || (request.headers.get('Accept') || '').includes('text/markdown');
+  }
+
   // Forward Notion routing headers (e.g. x-notion-cell for fanout requests).
   // Without them Notion keeps answering with fanoutData and pages render as 404.
   function buildApiHeaders(request) {
@@ -916,6 +934,7 @@ ${
       return notFoundResponse(url, true);
     }
     let response;
+    const pageFetchHeaders = isCrawler(request) ? CRAWLER_FETCH_HEADERS : PAGE_FETCH_HEADERS;
     const matchedSlug = pathToSlug(url.pathname);
     const isAppJs = url.pathname.startsWith('/app') && url.pathname.endsWith('js');
     const isAssetJs = url.pathname.startsWith('/_assets/') && url.pathname.endsWith('.js');
@@ -1009,15 +1028,15 @@ ${
       const pageId = SLUG_TO_PAGE[matchedSlug];
       url.pathname = '/' + pageId;
       response = await fetch(url.toString(), {
-        headers: PAGE_FETCH_HEADERS,
+        headers: pageFetchHeaders,
       });
       response = new Response(response.body, response);
       response.headers.delete('Content-Security-Policy');
       response.headers.delete('X-Content-Security-Policy');
       return appendJavascript(response, SLUG_TO_PAGE, matchedSlug, url);
     } else {
       response = await fetch(url.toString(), {
-        headers: PAGE_FETCH_HEADERS,
+        headers: pageFetchHeaders,
       });
       // HTMLRewriter parses any body regardless of Content-Type, so stream
       // non-HTML responses (CSS, images, files) through untouched
@@ -1114,13 +1133,34 @@ ${
 
   class LinkRewriter {
     element(element) {
+      const rel = element.getAttribute('rel');
+      // The server-rendered page names its notion.site URL as canonical; HeadRewriter adds ours
+      if (rel === 'canonical') {
+        element.remove();
+        return;
+      }
       // Remove Notion's default favicon links when custom favicon is set (Issue #16)
-      if (FAVICON_URL !== '') {
-        const rel = element.getAttribute('rel');
-        if (rel && (rel.includes('icon') || rel === 'apple-touch-icon')) {
-          element.remove();
-        }
+      if (FAVICON_URL !== '' && rel && (rel.includes('icon') || rel === 'apple-touch-icon')) {
+        element.remove();
+      }
+    }
+  }
+
+  // Links in the server-rendered page point to notion.site. Keep them on this
+  // domain, with the slug for mapped pages.
+  class AnchorRewriter {
+    element(element) {
+      let link;
+      try {
+        link = new URL(element.getAttribute('href'));
+      } catch (e) {
+        return;
       }
+      if (link.hostname !== NOTION_SITE_DOMAIN) return;
+      link.hostname = MY_DOMAIN;
+      const pageId = link.pathname.match(/[0-9a-f]{32}$/)?.[0];
+      if (pageId && PAGE_TO_SLUG[pageId] !== undefined) link.pathname = '/' + PAGE_TO_SLUG[pageId];
+      element.setAttribute('href', link.toString());
     }
   }
 
@@ -1587,6 +1627,7 @@ ${
       .on('title', metaRewriter)
       .on('meta', metaRewriter)
       .on('link', linkRewriter)
+      .on('a[href]', new AnchorRewriter())
       .on('head', headRewriter)
       .on('body', new BodyRewriter(SLUG_TO_PAGE, res.status === 404))
       .transform(res);

test/generated-worker.test.mjs

@@ -136,6 +136,33 @@ test('malformed JSON passes through without fabricated site context', async (t)
   assert.equal(forwarded.body, '{invalid');
 });
 
+// Returns the user agent the worker sends to Notion for a page request
+async function pageFetchUserAgent(t, headers) {
+  globalThis.HTMLRewriter = class { on() { return this; } transform(response) { return response; } };
+  t.after(() => { delete globalThis.HTMLRewriter; });
+  const upstream = t.mock.method(globalThis, 'fetch', async () => new Response('<html></html>', {
+    headers: { 'Content-Type': 'text/html' },
+  }));
+  await run(worker, new Request('https://notes.example.com/chapter', { headers }));
+  return new Headers(upstream.mock.calls[0].arguments[1].headers).get('user-agent');
+}
+
+const WORKNOT_BOT = 'Mozilla/5.0 (compatible; WorknotBot/1.0; +https://worknot.classmethod.live/)';
+const CHROME = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36';
+
+for (const [visitor, headers, expected] of [
+  ['GPTBot', { 'User-Agent': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)' }, WORKNOT_BOT],
+  ['ChatGPT-User', { 'User-Agent': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot' }, WORKNOT_BOT],
+  ['Googlebot', { 'User-Agent': 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' }, WORKNOT_BOT],
+  ['an agent asking for Markdown', { 'User-Agent': 'curl/8.7.1', Accept: 'text/markdown' }, WORKNOT_BOT],
+  ['a browser', { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36' }, CHROME],
+  ['a CUBOT phone', { 'User-Agent': 'Mozilla/5.0 (Linux; Android 10; CUBOT X30) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Mobile Safari/537.36' }, CHROME],
+]) {
+  test(`pages for ${visitor} are fetched as ${expected === WORKNOT_BOT ? 'a crawler' : 'a browser'}`, async (t) => {
+    assert.equal(await pageFetchUserAgent(t, headers), expected);
+  });
+}
+
 test('cross-origin preflight requests get no CORS headers', async (t) => {
   const upstream = t.mock.method(globalThis, 'fetch', async () => new Response());
   const response = await run(worker, new Request('https://notes.example.com/api/v3/getPublicPageData', {