From ee265892443d64a73d61074380fc7f1d84ed1bfb Mon Sep 17 00:00:00 2001 From: shigahi Date: Wed, 28 Oct 2026 11:27:15 +0100 Subject: [PATCH] Add Content Signals to the generated robots.txt Let users set the search, ai-input and ai-train Content Signals in the SEO settings. The worker adds a Content-Signal line to the robots.txt group for all crawlers, creating that group if the custom rules have none. A separate group could hide the user's rules from parsers that only read the first matching group. Closes #187 Co-Authored-By: Claude Opus 5.5 --- src/App.tsx | 36 +++++++++++++++++++++ src/code.ts | 32 +++++++++++++++++- test/generated-worker.test.mjs | 59 +++++++++++++++++++++++++--------- 3 files changed, 111 insertions(+), 16 deletions(-) diff --git a/src/App.tsx b/src/App.tsx index a7e3f96..c0a02f4 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -53,6 +53,7 @@ import code, { RssOptions, I18nOptions, OgImageGenerationOptions, + CONTENT_SIGNALS, } from "./code"; const DEFAULT_DOMAIN = "www.example.com"; @@ -1136,6 +1137,41 @@ export default function App() { variant="outlined" size="small" /> + + {CONTENT_SIGNALS.map((signal) => ( + + {signal} + + + ))} + + + + Content Signals + {" "} + in robots.txt tell crawlers how they may use your content: + search results (search), AI answers (ai-input) and AI training + (ai-train). Compliance is voluntary; to block crawlers, use AI + Crawl Control in your Cloudflare dashboard. + diff --git a/src/code.ts b/src/code.ts index 9358013..a32d4a0 100644 --- a/src/code.ts +++ b/src/code.ts @@ -45,9 +45,14 @@ export interface SocialPreviewOptions { locale?: string; } +// Content Signals (https://contentsignals.org/), each set to "yes", "no" or left out +export const CONTENT_SIGNALS = ["search", "ai-input", "ai-train"] as const; +export type ContentSignal = (typeof CONTENT_SIGNALS)[number]; + export interface SeoOptions { aiAttribution?: string; robotsRules?: string; + contentSignals?: Partial>; } export interface AnalyticsOptions { @@ -169,6 +174,13 @@ function tpl(value: string | undefined): string { return "`" + escaped + "`"; } +// Format the set signals as a Content-Signal value, e.g. "search=yes, ai-train=no". +function contentSignal(signals: SeoOptions["contentSignals"]): string { + return CONTENT_SIGNALS.filter((name) => signals?.[name] === "yes" || signals?.[name] === "no") + .map((name) => `${name}=${signals?.[name]}`) + .join(", "); +} + // Short content hash, used to version the worker's edge cache. function hash(text: string): string { let h = 0x811c9dc5; @@ -269,9 +281,12 @@ ${slugs * Step 3.4: SEO configuration (optional) * AI attribution for proper citation in AI-generated content * robots.txt rules (the sitemap line is added automatically) + * Content Signals: how crawlers may use the content, e.g. "search=yes, ai-train=no" + * (https://contentsignals.org/) */ const AI_ATTRIBUTION = ${str(seo?.aiAttribution)}; const ROBOTS_RULES = ${tpl(seo?.robotsRules?.trim())}; + const CONTENT_SIGNAL = ${str(contentSignal(seo?.contentSignals))}; /* * Step 3.5: analytics configuration (optional) @@ -506,6 +521,21 @@ ${ return cacheControl.includes('immutable') || /max-age=[1-9]/.test(cacheControl); } + // Content Signals go into the group for all crawlers, which is added if the rules + // have none. A separate group could hide the rules from parsers that use the first match. + function generateRobotsTxt() { + let rules = ROBOTS_RULES || 'User-agent: *\\nAllow: /'; + if (CONTENT_SIGNAL) { + const signal = 'Content-Signal: ' + CONTENT_SIGNAL; + // The "User-agent: *" line and any user-agent lines right after it + const allCrawlers = /^user-agent:[ \\t]*\\*[ \\t]*$(?:\\r?\\n^user-agent:.*$)*/im; + rules = allCrawlers.test(rules) + ? rules.replace(allCrawlers, (group) => group + '\\n' + signal) + : 'User-agent: *\\n' + signal + '\\n\\n' + rules; + } + return rules + '\\n\\nSitemap: https://' + MY_DOMAIN + '/sitemap.xml\\n'; + } + function generateSitemap() { let sitemap = ''; slugs.forEach( @@ -883,7 +913,7 @@ ${ // Use the original Notion site domain instead of www.notion.so url.hostname = NOTION_SITE_DOMAIN; if (url.pathname === '/robots.txt') { - return new Response((ROBOTS_RULES || 'User-agent: *\\nAllow: /') + '\\n\\nSitemap: https://' + MY_DOMAIN + '/sitemap.xml\\n'); + return new Response(generateRobotsTxt()); } if (url.pathname === '/sitemap.xml') { let response = new Response(generateSitemap()); diff --git a/test/generated-worker.test.mjs b/test/generated-worker.test.mjs index 3edc286..03fc45b 100644 --- a/test/generated-worker.test.mjs +++ b/test/generated-worker.test.mjs @@ -7,30 +7,30 @@ const ROOT = '11111111111111111111111111111111'; const CHAPTER = '22222222222222222222222222222222'; const OTHER = '33333333333333333333333333333333'; const dashed = (id) => id.replace(/(.{8})(.{4})(.{4})(.{4})(.{12})/, '$1-$2-$3-$4-$5'); -let script; +const CONFIG = { + myDomain: 'notes.example.com', + notionUrl: `https://source-workspace.notion.site/${ROOT}`, + slugs: [['chapter', `https://source-workspace.notion.site/${OTHER}`]], + pageTitle: '', pageDescription: '', googleFont: '', customScript: '', customCss: '', + optionImage: {}, pageMetadata: {}, structuredData: { enabled: false, schemaType: 'WebPage' }, + branding: {}, socialPreview: {}, seo: {}, analytics: {}, caching: { enabled: false }, + customHtml: {}, custom404: {}, subdomainRedirects: [], redirectRules: [], + rss: { enabled: false }, i18n: { enabled: false, defaultLocale: 'en' }, + ogImageGeneration: { enabled: false }, +}; +let generate; let worker; let isolates = 0; // Each import is a separate module instance, like a new isolate -async function loadWorker() { - const source = `${script}\n// isolate ${isolates++}`; +async function loadWorker(config = {}) { + const source = `${generate({ ...CONFIG, ...config })}\n// isolate ${isolates++}`; return (await import(`data:text/javascript;base64,${Buffer.from(source).toString('base64')}`)).default; } before(async () => { const vite = await createServer({ configFile: false, server: { middlewareMode: true, ws: false }, appType: 'custom', logLevel: 'error' }); try { - const { default: generate } = await vite.ssrLoadModule('/src/code.ts'); - script = generate({ - myDomain: 'notes.example.com', - notionUrl: `https://source-workspace.notion.site/${ROOT}`, - slugs: [['chapter', `https://source-workspace.notion.site/${OTHER}`]], - pageTitle: '', pageDescription: '', googleFont: '', customScript: '', customCss: '', - optionImage: {}, pageMetadata: {}, structuredData: { enabled: false, schemaType: 'WebPage' }, - branding: {}, socialPreview: {}, seo: {}, analytics: {}, caching: { enabled: false }, - customHtml: {}, custom404: {}, subdomainRedirects: [], redirectRules: [], - rss: { enabled: false }, i18n: { enabled: false, defaultLocale: 'en' }, - ogImageGeneration: { enabled: false }, - }); + ({ default: generate } = await vite.ssrLoadModule('/src/code.ts')); worker = await loadWorker(); } finally { await vite.close(); @@ -190,3 +190,32 @@ test('a failed foreign page check is not cached', async (t) => { const checks = upstream.mock.calls.filter((call) => call.arguments[0].endsWith('/api/v3/getPublicPageData')); assert.equal(checks.length, 2); }); + +async function robotsTxt(seo) { + const response = await run(await loadWorker({ seo }), new Request('https://notes.example.com/robots.txt')); + return response.text(); +} + +const SITEMAP = '\n\nSitemap: https://notes.example.com/sitemap.xml\n'; + +test('robots.txt is unchanged without Content Signals', async () => { + assert.equal(await robotsTxt({}), `User-agent: *\nAllow: /${SITEMAP}`); +}); + +test('Content Signals keep a fixed order and skip unset or invalid values', async () => { + const signals = { 'ai-train': 'no', 'ai-input': 'maybe', search: 'yes' }; + assert.equal(await robotsTxt({ contentSignals: signals }), + `User-agent: *\nContent-Signal: search=yes, ai-train=no\nAllow: /${SITEMAP}`); +}); + +test('Content Signals join the existing group for all crawlers', async () => { + const robotsRules = 'User-agent: GPTBot\nDisallow: /\n\nUser-agent: *\nUser-agent: Bingbot\nDisallow: /private'; + assert.equal(await robotsTxt({ robotsRules, contentSignals: { 'ai-train': 'no' } }), + `User-agent: GPTBot\nDisallow: /\n\nUser-agent: *\nUser-agent: Bingbot\nContent-Signal: ai-train=no\nDisallow: /private${SITEMAP}`); +}); + +test('Content Signals add a group for all crawlers when the rules have none', async () => { + const robotsRules = 'User-agent: GPTBot\nDisallow: /'; + assert.equal(await robotsTxt({ robotsRules, contentSignals: { 'ai-train': 'no' } }), + `User-agent: *\nContent-Signal: ai-train=no\n\nUser-agent: GPTBot\nDisallow: /${SITEMAP}`); +});