Update robots.txt to block crawling on non‑prod hosts

app/robots.ts

@@ -0,0 +1,28 @@
+import type { MetadataRoute } from "next"
+
+import { SITE_URL } from "@/lib/constants"
+
+export default function robots(): MetadataRoute.Robots {
+  let hostname = ""
+  try {
+    hostname = new URL(SITE_URL).hostname
+  } catch (error) {
+    console.error("Error getting hostname", error)
+  }
+
+  const isProduction = hostname === "ethereum.org"
+
+  if (!isProduction) {
+    return {
+      rules: [{ userAgent: "*", disallow: "/" }],
+      sitemap: [],
+      host: SITE_URL,
+    }
+  }
+
+  return {
+    rules: [{ userAgent: "*", allow: "/" }],
+    sitemap: [`${SITE_URL}/sitemap.xml`],
+    host: SITE_URL,
+  }
+}

next-sitemap.config.js

@@ -5,8 +5,7 @@ const defaultLocale = "en"
 
 /** @type {import('next-sitemap').IConfig} */
 module.exports = {
-  siteUrl: process.env.SITE_URL || "https://ethereum.org",
-  generateRobotsTxt: true,
+  siteUrl: process.env.NEXT_PUBLIC_SITE_URL || "https://ethereum.org",
   transform: async (_, path) => {
     const rootPath = path.split("/")[1]
     if (path.endsWith("/404")) return null

src/lib/ab-testing/server.ts

@@ -1,10 +1,8 @@
-import { SITE_URL } from "@/lib/constants"
-
 import type { ABTestAssignment, ABTestConfig } from "./types"
 
 const getABTestConfigs = async (): Promise<Record<string, ABTestConfig>> => {
   try {
-    const response = await fetch(`${SITE_URL}/api/ab-config`, {
+    const response = await fetch("https://ethereum.org/api/ab-config", {
       next: { revalidate: 3600 },
     })