fix(seo): prevent contributors.txt from indexing and crawling (#11149)

* chore(article-footer): set rel="nofollow" on contributors.txt links

* chore(cloud-function): set X-Robots-Tag header on /contributors.txt

* fix(robots): disallow contributors.txt

Author: caugner

Diff for: client/src/document/organisms/article-footer/index.tsx

@@ -29,7 +29,11 @@ export function LastModified({ value, locale }) {
 }
 
 export function Authors({ url }) {
-  return <a href={`${url}/contributors.txt`}>MDN contributors</a>;
+  return (
+    <a href={`${url}/contributors.txt`} rel="nofollow">
+      MDN contributors
+    </a>
+  );
 }
 
 enum ArticleFooterView {

Diff for: cloud-function/src/headers.ts

@@ -38,6 +38,10 @@ export function withContentResponseHeaders(
     xFrame: !isLiveSample,
   });
 
+  if (req.url?.endsWith("/contributors.txt")) {
+    res.setHeader("X-Robots-Tag", "noindex, nofollow");
+  }
+
   if (req.url?.endsWith("/sitemap.xml.gz")) {
     res.setHeader("Content-Type", "application/xml");
     res.setHeader("Content-Encoding", "gzip");

Diff for: tool/build-robots-txt.ts

@@ -13,6 +13,7 @@ User-agent: *
 Sitemap: https://developer.mozilla.org/sitemap.xml
 
 Disallow: /api/
+Disallow: /*/contributors.txt
 Disallow: /*/files/
 Disallow: /media
 `;