diff --git a/scripts/render-check.mjs b/scripts/render-check.mjs
index e32508e..d5a4e6e 100644
--- a/scripts/render-check.mjs
+++ b/scripts/render-check.mjs
@@ -23,16 +23,20 @@ const PORT = Number(process.env.PORT || 8799);
const RUNS = Number(process.env.RUNS || 2);
const DB_VIEWS = ["table", "board", "gallery", "list", "calendar", "timeline"];
+const CRAWLER = "Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)";
const TARGETS = {
worker: {
origin: `https://${DOMAIN}`,
+ // For requests outside the browser, which bypass its routing to wrangler
+ direct: `https://127.0.0.1:${PORT}`,
root: "/",
isRoot: (p) => p === "/",
isSubpage: (p) => p === "/subpage",
},
notion: {
origin: NOTION_SITE,
+ direct: NOTION_SITE,
root: `/${ROOT_PAGE}`,
isRoot: (p) => p.endsWith(ROOT_PAGE),
isSubpage: (p) => p.endsWith(SUBPAGE),
@@ -233,6 +237,19 @@ const CHECKS = {
s = await state(page);
return target.isSubpage(s.path) && s.h1 === "Render Check Subpage" ? null : `after forward: ${s.path} "${s.h1}"`;
},
+ // Crawlers get the server-rendered page, with the canonical URL and links on this site
+ "crawler content": async ({ context, target }) => {
+ const response = await context.request.get(target.direct + target.root, { headers: { "user-agent": CRAWLER } });
+ const html = await response.text();
+ if (!html.includes("Test page for Worknot")) return "page text missing";
+ const host = new URL(target.origin).host;
+ const canonical = [...html.matchAll(/ new URL(m[1]).host);
+ if (canonical.length !== 1 || canonical[0] !== host) return `canonical: ${canonical.join(", ") || "none"}`;
+ const offsite = [...html.matchAll(/]*href="(https:\/\/[^"]+\.notion\.site\/[^"]*)"/g)]
+ .map((m) => m[1])
+ .find((href) => new URL(href).host !== host);
+ return offsite ? `links to notion.site: ${offsite}` : null;
+ },
"no errors": async ({ errors, failures }) => {
const problems = [...errors, ...failures];
return problems.length ? problems.slice(0, 5).join("; ") : null;
@@ -240,7 +257,7 @@ const CHECKS = {
};
const GROUPS = [
- ["page renders", "database views", "database rows and formula", "images load", "pdf and file blocks", "mermaid diagram", "embeds and bookmarks", "anchor link", "no errors"],
+ ["page renders", "database views", "database rows and formula", "images load", "pdf and file blocks", "mermaid diagram", "embeds and bookmarks", "anchor link", "crawler content", "no errors"],
// Checks that navigate run on a page of their own
["database item stays on domain"],
["navigation and history"],
diff --git a/src/code.ts b/src/code.ts
index 9358013..14eec73 100644
--- a/src/code.ts
+++ b/src/code.ts
@@ -795,6 +795,24 @@ ${
'accept-language': 'en-US,en;q=0.9'
};
+ // Notion answers bot-like user agents with a server-rendered page that has the
+ // content as plain HTML. The visitor's user agent is not forwarded, because
+ // Notion rejects Googlebot or Bingbot requests that come from a Worker.
+ const CRAWLER_FETCH_HEADERS = {
+ ...PAGE_FETCH_HEADERS,
+ 'user-agent': 'Mozilla/5.0 (compatible; WorknotBot/1.0; +https://worknot.classmethod.live/)'
+ };
+
+ // The server-rendered page has no Notion app, so people must never get it:
+ // "bot" also matches CUBOT phones, which are excluded.
+ const CRAWLER_USER_AGENT = /bot\\b|crawler|spider|chatgpt-user|claude-user|perplexity-user|meta-externalagent/i;
+
+ function isCrawler(request) {
+ const userAgent = request.headers.get('User-Agent') || '';
+ return (CRAWLER_USER_AGENT.test(userAgent) && !/cubot/i.test(userAgent))
+ || (request.headers.get('Accept') || '').includes('text/markdown');
+ }
+
// Forward Notion routing headers (e.g. x-notion-cell for fanout requests).
// Without them Notion keeps answering with fanoutData and pages render as 404.
function buildApiHeaders(request) {
@@ -916,6 +934,7 @@ ${
return notFoundResponse(url, true);
}
let response;
+ const pageFetchHeaders = isCrawler(request) ? CRAWLER_FETCH_HEADERS : PAGE_FETCH_HEADERS;
const matchedSlug = pathToSlug(url.pathname);
const isAppJs = url.pathname.startsWith('/app') && url.pathname.endsWith('js');
const isAssetJs = url.pathname.startsWith('/_assets/') && url.pathname.endsWith('.js');
@@ -1009,7 +1028,7 @@ ${
const pageId = SLUG_TO_PAGE[matchedSlug];
url.pathname = '/' + pageId;
response = await fetch(url.toString(), {
- headers: PAGE_FETCH_HEADERS,
+ headers: pageFetchHeaders,
});
response = new Response(response.body, response);
response.headers.delete('Content-Security-Policy');
@@ -1017,7 +1036,7 @@ ${
return appendJavascript(response, SLUG_TO_PAGE, matchedSlug, url);
} else {
response = await fetch(url.toString(), {
- headers: PAGE_FETCH_HEADERS,
+ headers: pageFetchHeaders,
});
// HTMLRewriter parses any body regardless of Content-Type, so stream
// non-HTML responses (CSS, images, files) through untouched
@@ -1114,13 +1133,34 @@ ${
class LinkRewriter {
element(element) {
+ const rel = element.getAttribute('rel');
+ // The server-rendered page names its notion.site URL as canonical; HeadRewriter adds ours
+ if (rel === 'canonical') {
+ element.remove();
+ return;
+ }
// Remove Notion's default favicon links when custom favicon is set (Issue #16)
- if (FAVICON_URL !== '') {
- const rel = element.getAttribute('rel');
- if (rel && (rel.includes('icon') || rel === 'apple-touch-icon')) {
- element.remove();
- }
+ if (FAVICON_URL !== '' && rel && (rel.includes('icon') || rel === 'apple-touch-icon')) {
+ element.remove();
+ }
+ }
+ }
+
+ // Links in the server-rendered page point to notion.site. Keep them on this
+ // domain, with the slug for mapped pages.
+ class AnchorRewriter {
+ element(element) {
+ let link;
+ try {
+ link = new URL(element.getAttribute('href'));
+ } catch (e) {
+ return;
}
+ if (link.hostname !== NOTION_SITE_DOMAIN) return;
+ link.hostname = MY_DOMAIN;
+ const pageId = link.pathname.match(/[0-9a-f]{32}$/)?.[0];
+ if (pageId && PAGE_TO_SLUG[pageId] !== undefined) link.pathname = '/' + PAGE_TO_SLUG[pageId];
+ element.setAttribute('href', link.toString());
}
}
@@ -1587,6 +1627,7 @@ ${
.on('title', metaRewriter)
.on('meta', metaRewriter)
.on('link', linkRewriter)
+ .on('a[href]', new AnchorRewriter())
.on('head', headRewriter)
.on('body', new BodyRewriter(SLUG_TO_PAGE, res.status === 404))
.transform(res);
diff --git a/test/generated-worker.test.mjs b/test/generated-worker.test.mjs
index 3edc286..158f6fd 100644
--- a/test/generated-worker.test.mjs
+++ b/test/generated-worker.test.mjs
@@ -136,6 +136,33 @@ test('malformed JSON passes through without fabricated site context', async (t)
assert.equal(forwarded.body, '{invalid');
});
+// Returns the user agent the worker sends to Notion for a page request
+async function pageFetchUserAgent(t, headers) {
+ globalThis.HTMLRewriter = class { on() { return this; } transform(response) { return response; } };
+ t.after(() => { delete globalThis.HTMLRewriter; });
+ const upstream = t.mock.method(globalThis, 'fetch', async () => new Response('', {
+ headers: { 'Content-Type': 'text/html' },
+ }));
+ await run(worker, new Request('https://notes.example.com/chapter', { headers }));
+ return new Headers(upstream.mock.calls[0].arguments[1].headers).get('user-agent');
+}
+
+const WORKNOT_BOT = 'Mozilla/5.0 (compatible; WorknotBot/1.0; +https://worknot.classmethod.live/)';
+const CHROME = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36';
+
+for (const [visitor, headers, expected] of [
+ ['GPTBot', { 'User-Agent': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)' }, WORKNOT_BOT],
+ ['ChatGPT-User', { 'User-Agent': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot' }, WORKNOT_BOT],
+ ['Googlebot', { 'User-Agent': 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' }, WORKNOT_BOT],
+ ['an agent asking for Markdown', { 'User-Agent': 'curl/8.7.1', Accept: 'text/markdown' }, WORKNOT_BOT],
+ ['a browser', { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36' }, CHROME],
+ ['a CUBOT phone', { 'User-Agent': 'Mozilla/5.0 (Linux; Android 10; CUBOT X30) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Mobile Safari/537.36' }, CHROME],
+]) {
+ test(`pages for ${visitor} are fetched as ${expected === WORKNOT_BOT ? 'a crawler' : 'a browser'}`, async (t) => {
+ assert.equal(await pageFetchUserAgent(t, headers), expected);
+ });
+}
+
test('cross-origin preflight requests get no CORS headers', async (t) => {
const upstream = t.mock.method(globalThis, 'fetch', async () => new Response());
const response = await run(worker, new Request('https://notes.example.com/api/v3/getPublicPageData', {