fix(search): use "noindex, follow" robots tag + remove robots.txt exclude (#11140)

* fix(spas): set "noindex, follow" for search

Previously, it was "index, follow", but we do not want to index
MDN full-text search results.

* fix(robots): remove /search from robots.txt

Author: caugner

Diff for: build/spas.ts

@@ -140,7 +140,7 @@ export async function buildSPAs(options: {
 
       const SPAs = [
         { prefix: "play", pageTitle: "Playground | MDN" },
-        { prefix: "search", pageTitle: "Search" },
+        { prefix: "search", pageTitle: "Search", onlyFollow: true },
         { prefix: "plus", pageTitle: MDN_PLUS_TITLE },
         {
           prefix: "plus/ai-help",
@@ -177,12 +177,13 @@ export async function buildSPAs(options: {
         },
       ];
       const locale = VALID_LOCALES.get(pathLocale) || pathLocale;
-      for (const { prefix, pageTitle, noIndexing } of SPAs) {
+      for (const { prefix, pageTitle, noIndexing, onlyFollow } of SPAs) {
         const url = `/${locale}/${prefix}`;
         const context = {
           pageTitle,
           locale,
           noIndexing,
+          onlyFollow,
         };
 
         const html = renderCanonicalHTML(url, context);

Diff for: libs/types/hydration.ts

@@ -8,7 +8,8 @@ interface HydrationData<T = any, S = any> {
   pageTitle?: any;
   possibleLocales?: any;
   locale?: any;
-  noIndexing?: any;
+  noIndexing?: boolean;
+  onlyFollow?: boolean;
   image?: string | null;
 }
 

Diff for: ssr/render.ts

@@ -106,7 +106,8 @@ export default function render(
     pageTitle = null,
     possibleLocales = null,
     locale = null,
-    noIndexing = null,
+    noIndexing = false,
+    onlyFollow = false,
     image = null,
     blogMeta = null,
   }: HydrationData = {}
@@ -211,7 +212,9 @@ export default function render(
   const robotsContent =
     !ALWAYS_ALLOW_ROBOTS || (doc && doc.noIndexing) || noIndexing
       ? "noindex, nofollow"
-      : "";
+      : onlyFollow
+        ? "noindex, follow"
+        : "";
   const robotsMeta = robotsContent
     ? `<meta name="robots" content="${robotsContent}">`
     : "";

Diff for: tool/build-robots-txt.ts

@@ -5,7 +5,6 @@
  */
 import fs from "node:fs";
 
-import { VALID_LOCALES } from "../libs/constants/index.js";
 import { ALWAYS_ALLOW_ROBOTS } from "../libs/env/index.js";
 
 const ALLOW_TEXT = `
@@ -25,12 +24,6 @@ Disallow: /
 `;
 
 export async function runBuildRobotsTxt(outfile: string) {
-  let content = ALWAYS_ALLOW_ROBOTS ? ALLOW_TEXT : DISALLOW_TEXT;
-  if (ALWAYS_ALLOW_ROBOTS) {
-    // Append extra lines specifically when we do allow robots.
-    for (const locale of VALID_LOCALES.values()) {
-      content += `Disallow: /${locale}/search\n`;
-    }
-  }
+  const content = ALWAYS_ALLOW_ROBOTS ? ALLOW_TEXT : DISALLOW_TEXT;
   fs.writeFileSync(outfile, `${content.trim()}\n`, "utf-8");
 }