feat(dev): unstable dev server improvements

.changeset/dev-server.md

@@ -0,0 +1,156 @@
+---
+"@remix-run/dev": minor
+"@remix-run/server-runtime": minor
+---
+
+Dev server improvements
+
+- Push-based app server syncing that doesn't rely on polling
+- App server as a managed subprocess
+
+# Guide
+
+## 1. Enable new dev server
+
+Enable `unstable_dev` in `remix.config.js`:
+
+```js
+{
+  future: {
+    "unstable_dev": true
+  }
+}
+```
+
+## 2. Update `package.json` scripts
+
+Specify the command to run your app server with the `-c`/`--command` flag:
+
+For Remix app server:
+
+```json
+{
+  "scripts": {
+    "dev": "NODE_ENV=development remix dev -c 'node_modules/.bin/remix-serve build'"
+  }
+}
+```
+
+For any other servers, specify the command you use to run your production server.
+
+```json
+{
+  "scripts": {
+    "dev": "NODE_ENV=development remix dev -c 'node ./server.js'"
+  }
+}
+```
+
+## 3. Call `ping` in your app server
+
+For example, in an Express server:
+
+```js
+// server.mjs
+import path from "node:path";
+
+import express from "express";
+import { createRequestHandler } from "@remix-run/express";
+import { ping } from "@remix-run/dev";
+
+let BUILD_DIR = path.join(process.cwd(), "build"); // path to Remix's server build directory (`build/` by default)
+
+let app = express();
+
+app.all(
+  "*",
+  createRequestHandler({
+    build: require(BUILD_DIR),
+    mode: process.env.NODE_ENV,
+  })
+);
+
+app.listen(3000, () => {
+  let build = require(BUILD_DIR);
+  console.log('Ready: http://localhost:' + port);
+
+  // in development, call `ping` _after_ your server is ready
+  if (process.env.NODE_ENV === 'development') {
+    ping(build);
+  }
+});
+```
+
+## 4. That's it!
+
+You should now be able to run the Remix Dev server:
+
+```sh
+$ npm run dev
+# Ready: http://localhost:3000
+```
+
+Make sure you navigate to your app server's URL in the browser, in this example `http://localhost:3000`.
+Note: Any ports configured for the dev server are internal only (e.g. `--http-port` and `--websocket-port`)
+
+# Configuration
+
+Example:
+
+```js
+{
+  future: {
+    unstable_dev: {
+      // Port internally used by the dev server to receive app server `ping`s
+      httpPort: 3001, // by default, Remix chooses an open port in the range 3001-3099
+      // Port internally used by the dev server to send live reload, HMR, and HDR updates to the browser
+      websocketPort: 3002, // by default, Remix chooses an open port in the range 3001-3099
+      // Whether the app server should be restarted when app is rebuilt
+      // See `Advanced > restart` for more
+      restart: false, // default: `true`
+    }
+  }
+}
+```
+
+You can also configure via flags:
+
+```sh
+remix dev -c 'node ./server.mjs' --http-port=3001 --websocket-port=3002 --no-restart
+```
+
+## Advanced
+
+### Dev server scheme/host/port
+
+If you've customized the dev server's origin (e.g. for Docker or SSL support), you can use the `ping` options to specify the scheme/host/port for the dev server:
+
+```js
+ping(build, {
+  scheme: "https", // defaults to http
+  host: "mycustomhost", // defaults to localhost
+  port: 3003 // defaults to REMIX_DEV_HTTP_PORT environment variable
+});
+```
+
+### restart
+
+If you want to manage app server updates yourself, you can use the `--no-restart` flag so that the Remix dev server doesn't restart the app server subprocess when a rebuild succeeds.
+
+For example, if you rely on require cache purging to keep your app server running while server changes are pulled in, then you'll want to use `--no-restart`.
+
+🚨 It is then your responsibility to call `ping` whenever server changes are incorporated in your app server. 🚨
+
+So for require cache purging, you'd want to:
+1. Purge the require cache
+2. `require` your server build
+3. Call `ping` within a `if (process.env.NODE_ENV === 'development')`
+
+([Looking at you, Kent](https://github.com/kentcdodds/kentcdodds.com/blob/main/server/index.ts#L298) 😆)
+
+---
+
+The ultimate solution here would be to implement _server-side_ HMR (not to be confused with the more popular client-side HMR).
+Then your app server could continuously update itself with new build with 0 downtime and without losing in-memory data that wasn't affected by the server changes.
+
+That's left as an exercise to the reader.

integration/hmr-test.ts

@@ -7,15 +7,21 @@ import getPort, { makeRange } from "get-port";
 
 import { createFixtureProject, css, js, json } from "./helpers/create-fixture";
 
-let fixture = (options: { port: number; appServerPort: number }) => ({
+test.setTimeout(120_000);
+
+let fixture = (options: {
+  appServerPort: number;
+  httpPort: number;
+  webSocketPort: number;
+}) => ({
   files: {
     "remix.config.js": js`
       module.exports = {
         tailwind: true,
         future: {
           unstable_dev: {
-            port: ${options.port},
-            appServerPort: ${options.appServerPort},
+            httpPort: ${options.httpPort},
+            webSocketPort: ${options.webSocketPort},
           },
           v2_routeConvention: true,
           v2_errorBoundary: true,
@@ -28,8 +34,7 @@ let fixture = (options: { port: number; appServerPort: number }) => ({
       private: true,
       sideEffects: false,
       scripts: {
-        "dev:remix": `cross-env NODE_ENV=development node ./node_modules/@remix-run/dev/dist/cli.js dev`,
-        "dev:app": `cross-env NODE_ENV=development nodemon --watch build/ ./server.js`,
+        dev: `cross-env NODE_ENV=development node ./node_modules/@remix-run/dev/dist/cli.js dev -c "node ./server.js"`,
       },
       dependencies: {
         "@remix-run/css-bundle": "0.0.0-local-version",
@@ -38,7 +43,6 @@ let fixture = (options: { port: number; appServerPort: number }) => ({
         "cross-env": "0.0.0-local-version",
         express: "0.0.0-local-version",
         isbot: "0.0.0-local-version",
-        nodemon: "0.0.0-local-version",
         react: "0.0.0-local-version",
         "react-dom": "0.0.0-local-version",
         tailwindcss: "0.0.0-local-version",
@@ -58,6 +62,7 @@ let fixture = (options: { port: number; appServerPort: number }) => ({
       let path = require("path");
       let express = require("express");
       let { createRequestHandler } = require("@remix-run/express");
+      let { ping } = require("@remix-run/dev");
 
       const app = express();
       app.use(express.static("public", { immutable: true, maxAge: "1y" }));
@@ -75,8 +80,11 @@ let fixture = (options: { port: number; appServerPort: number }) => ({
 
       let port = ${options.appServerPort};
       app.listen(port, () => {
-        require(BUILD_DIR);
+        let build = require(BUILD_DIR);
         console.log('✅ app ready: http://localhost:' + port);
+        if (process.env.NODE_ENV === 'development') {
+          ping(build);
+        }
       });
     `,
 
@@ -204,43 +212,34 @@ let bufferize = (stream: Readable): (() => string) => {
   return () => buffer;
 };
 
+let HMR_TIMEOUT_MS = 10_000;
+
 test("HMR", async ({ page }) => {
   // uncomment for debugging
   // page.on("console", (msg) => console.log(msg.text()));
   page.on("pageerror", (err) => console.log(err.message));
 
-  let appServerPort = await getPort({ port: makeRange(3080, 3089) });
-  let port = await getPort({ port: makeRange(3090, 3099) });
-  let projectDir = await createFixtureProject(fixture({ port, appServerPort }));
+  let portRange = makeRange(3080, 3099);
+  let appServerPort = await getPort({ port: portRange });
+  let httpPort = await getPort({ port: portRange });
+  let webSocketPort = await getPort({ port: portRange });
+  let projectDir = await createFixtureProject(
+    fixture({ appServerPort, httpPort, webSocketPort })
+  );
 
   // spin up dev server
-  let dev = execa("npm", ["run", "dev:remix"], { cwd: projectDir });
+  let dev = execa("npm", ["run", "dev"], { cwd: projectDir });
   let devStdout = bufferize(dev.stdout!);
   let devStderr = bufferize(dev.stderr!);
   await wait(
     () => {
       let stderr = devStderr();
       if (stderr.length > 0) throw Error(stderr);
-      return /💿 Built in /.test(devStdout());
+      return /✅ app ready: /.test(devStdout());
     },
     { timeoutMs: 10_000 }
   );
 
-  // spin up app server
-  let app = execa("npm", ["run", "dev:app"], { cwd: projectDir });
-  let appStdout = bufferize(app.stdout!);
-  let appStderr = bufferize(app.stderr!);
-  await wait(
-    () => {
-      let stderr = appStderr();
-      if (stderr.length > 0) throw Error(stderr);
-      return /✅ app ready: /.test(appStdout());
-    },
-    {
-      timeoutMs: 10_000,
-    }
-  );
-
   try {
     await page.goto(`http://localhost:${appServerPort}`, {
       waitUntil: "networkidle",
@@ -290,7 +289,7 @@ test("HMR", async ({ page }) => {
     // detect HMR'd content and style changes
     await page.waitForLoadState("networkidle");
     let h1 = page.getByText("Changed");
-    await h1.waitFor({ timeout: 2000 });
+    await h1.waitFor({ timeout: HMR_TIMEOUT_MS });
     expect(h1).toHaveCSS("color", "rgb(255, 255, 255)");
     expect(h1).toHaveCSS("background-color", "rgb(0, 0, 0)");
 
@@ -301,7 +300,7 @@ test("HMR", async ({ page }) => {
     // undo change
     fs.writeFileSync(indexPath, originalIndex);
     fs.writeFileSync(cssModulePath, originalCssModule);
-    await page.getByText("Index Title").waitFor({ timeout: 2000 });
+    await page.getByText("Index Title").waitFor({ timeout: HMR_TIMEOUT_MS });
     expect(await page.getByLabel("Root Input").inputValue()).toBe("asdfasdf");
     await page.waitForSelector(`#root-counter:has-text("inc 1")`);
 
@@ -322,7 +321,7 @@ test("HMR", async ({ page }) => {
       }
     `;
     fs.writeFileSync(indexPath, withLoader1);
-    await page.getByText("Hello, world").waitFor({ timeout: 2000 });
+    await page.getByText("Hello, world").waitFor({ timeout: HMR_TIMEOUT_MS });
     expect(await page.getByLabel("Root Input").inputValue()).toBe("asdfasdf");
     await page.waitForSelector(`#root-counter:has-text("inc 1")`);
 
@@ -344,7 +343,7 @@ test("HMR", async ({ page }) => {
       }
     `;
     fs.writeFileSync(indexPath, withLoader2);
-    await page.getByText("Hello, planet").waitFor({ timeout: 2000 });
+    await page.getByText("Hello, planet").waitFor({ timeout: HMR_TIMEOUT_MS });
     expect(await page.getByLabel("Root Input").inputValue()).toBe("asdfasdf");
     await page.waitForSelector(`#root-counter:has-text("inc 1")`);
 
@@ -388,10 +387,16 @@ test("HMR", async ({ page }) => {
     aboutCounter = await page.waitForSelector(
       `#about-counter:has-text("inc 0")`
     );
+  } catch (e) {
+    console.log("stdout begin -----------------------");
+    console.log(devStdout());
+    console.log("stdout end -------------------------");
+
+    console.log("stderr begin -----------------------");
+    console.log(devStderr());
+    console.log("stderr end -------------------------");
+    throw e;
   } finally {
     dev.kill();
-    app.kill();
-    console.log(devStderr());
-    console.log(appStderr());
   }
 });

package.json

@@ -108,7 +108,6 @@
     "jest-watch-typeahead": "^0.6.5",
     "jsonfile": "^6.0.1",
     "lodash": "^4.17.21",
-    "nodemon": "^2.0.20",
     "npm-run-all": "^4.1.5",
     "patch-package": "^6.5.0",
     "prettier": "2.7.1",

packages/remix-dev/channel.ts

@@ -1,8 +1,8 @@
+import type { Result } from "./result";
+
 type Resolve<V> = (value: V | PromiseLike<V>) => void;
 type Reject = (reason?: any) => void;
 
-type Result<V, E = unknown> = { ok: true; value: V } | { ok: false; error: E };
-
 export type Type<V, E = unknown> = {
   ok: (value: V) => void;
   err: (reason?: any) => void;