feat: add isBot option to renderRouterToStream for streaming SSR bot detection

.changeset/ssr-isbot-option.md

@@ -0,0 +1,16 @@
+---
+'@tanstack/router-core': minor
+'@tanstack/react-router': minor
+'@tanstack/solid-router': minor
+'@tanstack/vue-router': minor
+---
+
+feat: add `ssr.isBot` router option to configure streaming SSR bot detection
+
+Streaming SSR waits for the full document (React's `allReady`) for requests that `isbot` flags by their `User-Agent`, and streams the shell first for everyone else. This was hardcoded, so performance auditors (Lighthouse, PageSpeed Insights, WebPageTest, …) — which `isbot` classifies as bots — were measured on the buffered path instead of the streaming path real users get.
+
+The new `ssr.isBot` router option overrides that decision while keeping the current behavior by default:
+
+- `undefined` (default): unchanged — use the built-in `isbot` User-Agent check.
+- `boolean`: force the bot (`true`) or non-bot (`false`) path for every request.
+- `(request) => boolean`: provide a custom predicate.

packages/react-router/src/ssr/renderRouterToStream.tsx

@@ -41,6 +41,15 @@ const isAbortError = (request: Request, error: unknown) =>
   (request.signal.aborted && error === request.signal.reason) ||
   (error instanceof Error && error.name === 'AbortError')
 
+// `ssr.isBot` lets apps override the default `isbot` User-Agent check that
+// decides whether to wait for the full document before streaming.
+const resolveIsBot = (router: AnyRouter, request: Request): boolean => {
+  const isBot = router.options.ssr?.isBot
+  if (typeof isBot === 'function') return isBot(request)
+  if (typeof isBot === 'boolean') return isBot
+  return isbot(request.headers.get('User-Agent'))
+}
+
 export const renderRouterToStream = async ({
   request,
   router,
@@ -52,6 +61,8 @@ export const renderRouterToStream = async ({
   responseHeaders: Headers
   children: ReactNode
 }) => {
+  const isBotRequest = resolveIsBot(router, request)
+
   if (typeof ReactDOMServer.renderToReadableStream === 'function') {
     const stream = await ReactDOMServer.renderToReadableStream(children, {
       signal: request.signal,
@@ -64,7 +75,7 @@ export const renderRouterToStream = async ({
       },
     })
 
-    if (isbot(request.headers.get('User-Agent'))) {
+    if (isBotRequest) {
       await waitForReadyOrAbort(stream.allReady, request.signal)
     }
 
@@ -150,7 +161,7 @@ export const renderRouterToStream = async ({
       pipeable = ReactDOMServer.renderToPipeableStream(children, {
         nonce: router.options.ssr?.nonce,
         progressiveChunkSize: Number.POSITIVE_INFINITY,
-        ...(isbot(request.headers.get('User-Agent'))
+        ...(isBotRequest
           ? {
               onAllReady() {
                 pipeable!.pipe(reactAppPassthrough)

packages/react-router/tests/renderRouterToStream.test.tsx

@@ -20,11 +20,14 @@ afterEach(() => {
   vi.restoreAllMocks()
 })
 
-async function buildRouter() {
+async function buildRouter(ssr?: {
+  isBot?: boolean | ((request: Request) => boolean)
+}) {
   const rootRoute = createRootRoute({ component: () => null })
   const router = createRouter({
     history: createMemoryHistory({ initialEntries: ['/'] }),
     routeTree: rootRoute,
+    ...(ssr ? { ssr } : {}),
   })
   router.isServer = true
   attachRouterServerSsrUtils({ router, manifest: undefined })
@@ -232,3 +235,89 @@ describe('renderRouterToStream - pipeable sync errors', () => {
     }
   })
 })
+
+describe('renderRouterToStream - bot detection (ssr.isBot)', () => {
+  const BOT_UA = 'Googlebot/2.1 (+http://www.google.com/bot.html)'
+  const HUMAN_UA =
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'
+
+  function requestWith(headers: Record<string, string>) {
+    return new Request('http://localhost/', { headers })
+  }
+
+  // Captures whether renderToPipeableStream was given `onAllReady` (bot, wait
+  // for the full document) or `onShellReady` (stream the shell first).
+  async function getStreamMode(
+    router: Awaited<ReturnType<typeof buildRouter>>,
+    request: Request,
+  ): Promise<'allReady' | 'shellReady' | undefined> {
+    let mode: 'allReady' | 'shellReady' | undefined
+    reactDomServerMocks.renderToPipeableStream.mockImplementationOnce(
+      (_children, opts) => {
+        mode = opts.onAllReady ? 'allReady' : 'shellReady'
+        queueMicrotask(() => (opts.onAllReady ?? opts.onShellReady)())
+        return { abort: vi.fn(), pipe: vi.fn() }
+      },
+    )
+
+    const result = await renderRouterToStream({
+      request,
+      router,
+      responseHeaders: new Headers(),
+      children: null,
+    })
+    try {
+      return mode
+    } finally {
+      await unwrapResponse(result)
+        .body?.cancel()
+        .catch(() => {})
+      router.serverSsr?.cleanup()
+    }
+  }
+
+  test('default: bot User-Agent waits for allReady', async () => {
+    const mode = await getStreamMode(
+      await buildRouter(),
+      requestWith({ 'user-agent': BOT_UA }),
+    )
+    expect(mode).toBe('allReady')
+  })
+
+  test('default: human User-Agent streams the shell', async () => {
+    const mode = await getStreamMode(
+      await buildRouter(),
+      requestWith({ 'user-agent': HUMAN_UA }),
+    )
+    expect(mode).toBe('shellReady')
+  })
+
+  test('ssr.isBot=false: bot User-Agent still streams the shell', async () => {
+    const mode = await getStreamMode(
+      await buildRouter({ isBot: false }),
+      requestWith({ 'user-agent': BOT_UA }),
+    )
+    expect(mode).toBe('shellReady')
+  })
+
+  test('ssr.isBot=true: human User-Agent waits for allReady', async () => {
+    const mode = await getStreamMode(
+      await buildRouter({ isBot: true }),
+      requestWith({ 'user-agent': HUMAN_UA }),
+    )
+    expect(mode).toBe('allReady')
+  })
+
+  test('ssr.isBot predicate receives the request and controls the mode', async () => {
+    const isBot = vi.fn(
+      (request: Request) => request.headers.get('x-prerender') === '1',
+    )
+    const router = await buildRouter({ isBot })
+    const request = requestWith({ 'user-agent': HUMAN_UA, 'x-prerender': '1' })
+
+    const mode = await getStreamMode(router, request)
+
+    expect(mode).toBe('allReady')
+    expect(isBot).toHaveBeenCalledWith(request)
+  })
+})

packages/router-core/src/router.ts

@@ -509,6 +509,20 @@ export interface RouterOptions<
   origin?: string
   ssr?: {
     nonce?: string
+    /**
+     * Determines whether a request should be treated as a bot/crawler.
+     *
+     * When a request is considered a bot, streaming SSR waits for the entire
+     * document to render (React's `allReady`) before responding, so crawlers
+     * receive complete HTML instead of a progressively streamed shell.
+     *
+     * - `undefined` (default): use the built-in `isbot` User-Agent check.
+     * - `boolean`: force the bot (`true`) or non-bot (`false`) path for every request.
+     * - `(request) => boolean`: provide a custom predicate.
+     *
+     * @default undefined
+     */
+    isBot?: boolean | ((request: Request) => boolean)
   }
 }
 

packages/solid-router/src/ssr/renderRouterToStream.tsx

@@ -11,6 +11,15 @@ import type { AnyRouter } from '@tanstack/router-core'
 
 const noop = () => {}
 
+// `ssr.isBot` lets apps override the default `isbot` User-Agent check that
+// decides whether to wait for the server renderer before streaming.
+const resolveIsBot = (router: AnyRouter, request: Request): boolean => {
+  const isBot = router.options.ssr?.isBot
+  if (typeof isBot === 'function') return isBot(request)
+  if (typeof isBot === 'boolean') return isBot
+  return isbot(request.headers.get('User-Agent'))
+}
+
 // Bot responses wait for the server renderer before streaming. If the request
 // disconnects during that wait, unblock so the pipe can abort and clean up.
 async function waitForReadyOrAbort(
@@ -44,6 +53,8 @@ export const renderRouterToStream = async ({
   responseHeaders: Headers
   children: () => JSXElement
 }) => {
+  const isBotRequest = resolveIsBot(router, request)
+
   const { writable, readable } = new TransformStream()
 
   const docType = Solid.ssr('<!DOCTYPE html>')
@@ -121,7 +132,7 @@ export const renderRouterToStream = async ({
     })
   }
 
-  if (isbot(request.headers.get('User-Agent'))) {
+  if (isBotRequest) {
     await waitForReadyOrAbort(
       Promise.resolve(stream as unknown),
       request.signal,

packages/vue-router/src/ssr/renderRouterToStream.tsx

@@ -14,6 +14,15 @@ const isAbortError = (request: Request, error: unknown) =>
   (error instanceof Error && error.name === 'AbortError') ||
   (error as any)?.code === 'ABORT_ERR'
 
+// `ssr.isBot` lets apps override the default `isbot` User-Agent check that
+// decides whether to wait for the full document before streaming.
+const resolveIsBot = (router: AnyRouter, request: Request): boolean => {
+  const isBot = router.options.ssr?.isBot
+  if (typeof isBot === 'function') return isBot(request)
+  if (typeof isBot === 'boolean') return isBot
+  return isbot(request.headers.get('User-Agent'))
+}
+
 function prependDoctype(
   readable: globalThis.ReadableStream,
 ): NodeReadableStream<Uint8Array> {
@@ -76,7 +85,9 @@ export const renderRouterToStream = async ({
 }) => {
   const app = Vue.createSSRApp(App, { router })
 
-  if (isbot(request.headers.get('User-Agent'))) {
+  const isBotRequest = resolveIsBot(router, request)
+
+  if (isBotRequest) {
     try {
       let cleanupAbortListener: (() => void) | undefined
       const abortPromise = new Promise<never>((_, reject) => {