Add Content Signals to the generated robots.txt

src/App.tsx

@@ -53,6 +53,7 @@ import code, {
   RssOptions,
   I18nOptions,
   OgImageGenerationOptions,
+  CONTENT_SIGNALS,
 } from "./code";
 
 const DEFAULT_DOMAIN = "www.example.com";
@@ -1136,6 +1137,41 @@ export default function App() {
               variant="outlined"
               size="small"
             />
+            <Stack direction="row" spacing={1}>
+              {CONTENT_SIGNALS.map((signal) => (
+                <FormControl key={signal} fullWidth size="small" margin="dense">
+                  <InputLabel id={`contentSignal-${signal}`}>{signal}</InputLabel>
+                  <Select
+                    labelId={`contentSignal-${signal}`}
+                    label={signal}
+                    value={seo.contentSignals?.[signal] ?? ""}
+                    onChange={(e) =>
+                      handleSeoChange("contentSignals", {
+                        ...seo.contentSignals,
+                        [signal]: e.target.value,
+                      })
+                    }
+                  >
+                    <MenuItem value="">Not set</MenuItem>
+                    <MenuItem value="yes">yes</MenuItem>
+                    <MenuItem value="no">no</MenuItem>
+                  </Select>
+                </FormControl>
+              ))}
+            </Stack>
+            <Typography variant="caption" color="textSecondary">
+              <Link
+                href="https://contentsignals.org/"
+                target="_blank"
+                rel="noreferrer"
+              >
+                Content Signals
+              </Link>{" "}
+              in robots.txt tell crawlers how they may use your content:
+              search results (search), AI answers (ai-input) and AI training
+              (ai-train). Compliance is voluntary; to block crawlers, use AI
+              Crawl Control in your Cloudflare dashboard.
+            </Typography>
           </Box>
 
           <Box sx={{ mt: 3, pt: 2, borderTop: 1, borderColor: "grey.300" }}>

src/code.ts

@@ -45,9 +45,14 @@ export interface SocialPreviewOptions {
   locale?: string;
 }
 
+// Content Signals (https://contentsignals.org/), each set to "yes", "no" or left out
+export const CONTENT_SIGNALS = ["search", "ai-input", "ai-train"] as const;
+export type ContentSignal = (typeof CONTENT_SIGNALS)[number];
+
 export interface SeoOptions {
   aiAttribution?: string;
   robotsRules?: string;
+  contentSignals?: Partial<Record<ContentSignal, string>>;
 }
 
 export interface AnalyticsOptions {
@@ -169,6 +174,13 @@ function tpl(value: string | undefined): string {
   return "`" + escaped + "`";
 }
 
+// Format the set signals as a Content-Signal value, e.g. "search=yes, ai-train=no".
+function contentSignal(signals: SeoOptions["contentSignals"]): string {
+  return CONTENT_SIGNALS.filter((name) => signals?.[name] === "yes" || signals?.[name] === "no")
+    .map((name) => `${name}=${signals?.[name]}`)
+    .join(", ");
+}
+
 // Short content hash, used to version the worker's edge cache.
 function hash(text: string): string {
   let h = 0x811c9dc5;
@@ -269,9 +281,12 @@ ${slugs
    * Step 3.4: SEO configuration (optional)
    * AI attribution for proper citation in AI-generated content
    * robots.txt rules (the sitemap line is added automatically)
+   * Content Signals: how crawlers may use the content, e.g. "search=yes, ai-train=no"
+   * (https://contentsignals.org/)
    */
   const AI_ATTRIBUTION = ${str(seo?.aiAttribution)};
   const ROBOTS_RULES = ${tpl(seo?.robotsRules?.trim())};
+  const CONTENT_SIGNAL = ${str(contentSignal(seo?.contentSignals))};
 
   /*
    * Step 3.5: analytics configuration (optional)
@@ -506,6 +521,21 @@ ${
     return cacheControl.includes('immutable') || /max-age=[1-9]/.test(cacheControl);
   }
 
+  // Content Signals go into the group for all crawlers, which is added if the rules
+  // have none. A separate group could hide the rules from parsers that use the first match.
+  function generateRobotsTxt() {
+    let rules = ROBOTS_RULES || 'User-agent: *\\nAllow: /';
+    if (CONTENT_SIGNAL) {
+      const signal = 'Content-Signal: ' + CONTENT_SIGNAL;
+      // The "User-agent: *" line and any user-agent lines right after it
+      const allCrawlers = /^user-agent:[ \\t]*\\*[ \\t]*$(?:\\r?\\n^user-agent:.*$)*/im;
+      rules = allCrawlers.test(rules)
+        ? rules.replace(allCrawlers, (group) => group + '\\n' + signal)
+        : 'User-agent: *\\n' + signal + '\\n\\n' + rules;
+    }
+    return rules + '\\n\\nSitemap: https://' + MY_DOMAIN + '/sitemap.xml\\n';
+  }
+
   function generateSitemap() {
     let sitemap = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
     slugs.forEach(
@@ -883,7 +913,7 @@ ${
     // Use the original Notion site domain instead of www.notion.so
     url.hostname = NOTION_SITE_DOMAIN;
     if (url.pathname === '/robots.txt') {
-      return new Response((ROBOTS_RULES || 'User-agent: *\\nAllow: /') + '\\n\\nSitemap: https://' + MY_DOMAIN + '/sitemap.xml\\n');
+      return new Response(generateRobotsTxt());
     }
     if (url.pathname === '/sitemap.xml') {
       let response = new Response(generateSitemap());

test/generated-worker.test.mjs

@@ -7,30 +7,30 @@ const ROOT = '11111111111111111111111111111111';
 const CHAPTER = '22222222222222222222222222222222';
 const OTHER = '33333333333333333333333333333333';
 const dashed = (id) => id.replace(/(.{8})(.{4})(.{4})(.{4})(.{12})/, '$1-$2-$3-$4-$5');
-let script;
+const CONFIG = {
+  myDomain: 'notes.example.com',
+  notionUrl: `https://source-workspace.notion.site/${ROOT}`,
+  slugs: [['chapter', `https://source-workspace.notion.site/${OTHER}`]],
+  pageTitle: '', pageDescription: '', googleFont: '', customScript: '', customCss: '',
+  optionImage: {}, pageMetadata: {}, structuredData: { enabled: false, schemaType: 'WebPage' },
+  branding: {}, socialPreview: {}, seo: {}, analytics: {}, caching: { enabled: false },
+  customHtml: {}, custom404: {}, subdomainRedirects: [], redirectRules: [],
+  rss: { enabled: false }, i18n: { enabled: false, defaultLocale: 'en' },
+  ogImageGeneration: { enabled: false },
+};
+let generate;
 let worker;
 let isolates = 0;
 // Each import is a separate module instance, like a new isolate
-async function loadWorker() {
-  const source = `${script}\n// isolate ${isolates++}`;
+async function loadWorker(config = {}) {
+  const source = `${generate({ ...CONFIG, ...config })}\n// isolate ${isolates++}`;
   return (await import(`data:text/javascript;base64,${Buffer.from(source).toString('base64')}`)).default;
 }
 
 before(async () => {
   const vite = await createServer({ configFile: false, server: { middlewareMode: true, ws: false }, appType: 'custom', logLevel: 'error' });
   try {
-    const { default: generate } = await vite.ssrLoadModule('/src/code.ts');
-    script = generate({
-      myDomain: 'notes.example.com',
-      notionUrl: `https://source-workspace.notion.site/${ROOT}`,
-      slugs: [['chapter', `https://source-workspace.notion.site/${OTHER}`]],
-      pageTitle: '', pageDescription: '', googleFont: '', customScript: '', customCss: '',
-      optionImage: {}, pageMetadata: {}, structuredData: { enabled: false, schemaType: 'WebPage' },
-      branding: {}, socialPreview: {}, seo: {}, analytics: {}, caching: { enabled: false },
-      customHtml: {}, custom404: {}, subdomainRedirects: [], redirectRules: [],
-      rss: { enabled: false }, i18n: { enabled: false, defaultLocale: 'en' },
-      ogImageGeneration: { enabled: false },
-    });
+    ({ default: generate } = await vite.ssrLoadModule('/src/code.ts'));
     worker = await loadWorker();
   } finally {
     await vite.close();
@@ -190,3 +190,32 @@ test('a failed foreign page check is not cached', async (t) => {
   const checks = upstream.mock.calls.filter((call) => call.arguments[0].endsWith('/api/v3/getPublicPageData'));
   assert.equal(checks.length, 2);
 });
+
+async function robotsTxt(seo) {
+  const response = await run(await loadWorker({ seo }), new Request('https://notes.example.com/robots.txt'));
+  return response.text();
+}
+
+const SITEMAP = '\n\nSitemap: https://notes.example.com/sitemap.xml\n';
+
+test('robots.txt is unchanged without Content Signals', async () => {
+  assert.equal(await robotsTxt({}), `User-agent: *\nAllow: /${SITEMAP}`);
+});
+
+test('Content Signals keep a fixed order and skip unset or invalid values', async () => {
+  const signals = { 'ai-train': 'no', 'ai-input': 'maybe', search: 'yes' };
+  assert.equal(await robotsTxt({ contentSignals: signals }),
+    `User-agent: *\nContent-Signal: search=yes, ai-train=no\nAllow: /${SITEMAP}`);
+});
+
+test('Content Signals join the existing group for all crawlers', async () => {
+  const robotsRules = 'User-agent: GPTBot\nDisallow: /\n\nUser-agent: *\nUser-agent: Bingbot\nDisallow: /private';
+  assert.equal(await robotsTxt({ robotsRules, contentSignals: { 'ai-train': 'no' } }),
+    `User-agent: GPTBot\nDisallow: /\n\nUser-agent: *\nUser-agent: Bingbot\nContent-Signal: ai-train=no\nDisallow: /private${SITEMAP}`);
+});
+
+test('Content Signals add a group for all crawlers when the rules have none', async () => {
+  const robotsRules = 'User-agent: GPTBot\nDisallow: /';
+  assert.equal(await robotsTxt({ robotsRules, contentSignals: { 'ai-train': 'no' } }),
+    `User-agent: *\nContent-Signal: ai-train=no\n\nUser-agent: GPTBot\nDisallow: /${SITEMAP}`);
+});