fix(ssr): remove unnecessary robots tags (#11139)

* fix(ssr): remove unnecessary robots tags

1. Robots tag for 404 page is unnecessary, as it is ignored anyways.
2. Robots tag with value "index, follow" is unnecessary, as it is the default.

* test(spas): update expectations

Author: caugner

Diff for: ssr/render.ts

@@ -209,13 +209,12 @@ export default function render(
   }</script>`;
 
   const robotsContent =
-    !ALWAYS_ALLOW_ROBOTS ||
-    (doc && doc.noIndexing) ||
-    pageNotFound ||
-    noIndexing
+    !ALWAYS_ALLOW_ROBOTS || (doc && doc.noIndexing) || noIndexing
       ? "noindex, nofollow"
-      : "index, follow";
-  const robotsMeta = `<meta name="robots" content="${robotsContent}">`;
+      : "";
+  const robotsMeta = robotsContent
+    ? `<meta name="robots" content="${robotsContent}">`
+    : "";
   const rssLink = `<link rel="alternate" type="application/rss+xml" title="MDN Blog RSS Feed" href="${BASE_URL}/${DEFAULT_LOCALE}/blog/rss.xml" hreflang="en" />`;
   const ssr_data = [...translations, ...WEBFONT_TAGS, rssLink, robotsMeta];
   let html = buildHtml;

Diff for: testing/tests/index.test.ts

@@ -119,7 +119,7 @@ test("content built foo page", () => {
     `https://developer.mozilla.org${doc.mdn_url}`
   );
 
-  expect($('meta[name="robots"]').attr("content")).toBe("index, follow");
+  expect($('meta[name="robots"]')).toHaveLength(0);
 
   // The HTML should contain the Google Analytics snippet.
   // The ID should match what's set in `.env.testing`.
@@ -1179,7 +1179,7 @@ test("404 page", () => {
   const $ = cheerio.load(html);
   expect($("title").text()).toContain("Page not found");
   expect($("h1").text()).toContain("Page not found");
-  expect($('meta[name="robots"]').attr("content")).toBe("noindex, nofollow");
+  expect($('meta[name="robots"]')).toHaveLength(0);
   expect($('meta[property="og:locale"]').attr("content")).toBe("en_US");
 });
 
@@ -1190,7 +1190,7 @@ test("plus page", () => {
   const html = fs.readFileSync(htmlFile, "utf-8");
   const $ = cheerio.load(html);
   expect($("title").text()).toContain("Plus");
-  expect($('meta[name="robots"]').attr("content")).toBe("index, follow");
+  expect($('meta[name="robots"]')).toHaveLength(0);
 });
 
 test("plus collections page", () => {