fix(relay): T3 Connect repairs tunnel ingress that drifted from a matching origin

infra/relay/src/environments/ManagedEndpointProvider.test.ts

@@ -4,6 +4,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices";
 import { describe, expect, it } from "@effect/vitest";
 import * as Alchemy from "alchemy";
 import * as Cloudflare from "@alchemy.run/cloudflare";
+import * as ZeroTrust from "@distilled.cloud/cloudflare/zero-trust";
 import * as Deferred from "effect/Deferred";
 import * as Effect from "effect/Effect";
 import * as Fiber from "effect/Fiber";
@@ -36,10 +37,25 @@ const config = RelayConfiguration.RelayConfiguration.of({
 });
 
 interface TunnelCall {
-  readonly operation: "get" | "list" | "create" | "putConfiguration" | "getToken" | "delete";
+  readonly operation:
+    | "get"
+    | "list"
+    | "create"
+    | "putConfiguration"
+    | "getConfiguration"
+    | "getToken"
+    | "delete";
   readonly input: unknown;
 }
 
+interface TunnelConfiguration {
+  readonly ingress: ReadonlyArray<{
+    readonly hostname?: string;
+    readonly path?: string;
+    readonly service: string;
+  }>;
+}
+
 interface DnsCall {
   readonly operation: "listRecords" | "createRecord" | "updateRecord" | "deleteRecord";
   readonly input: unknown;
@@ -88,6 +104,11 @@ function makeTunnelClient(calls: TunnelCall[] = []) {
       Effect.sync(() => {
         calls.push({ operation: "putConfiguration", input: { tunnelId, tunnelConfig } });
       }),
+    getConfiguration: (tunnelId) =>
+      Effect.sync(() => {
+        calls.push({ operation: "getConfiguration", input: tunnelId });
+        return { ingress: [] };
+      }),
     getToken: (tunnelId) =>
       Effect.sync(() => {
         calls.push({ operation: "getToken", input: tunnelId });
@@ -100,7 +121,10 @@ function makeTunnelClient(calls: TunnelCall[] = []) {
   });
 }
 
-function makePersistentTunnelClient(calls: TunnelCall[] = []) {
+function makePersistentTunnelClient(
+  calls: TunnelCall[] = [],
+  configuration: { current: TunnelConfiguration | null } = { current: null },
+) {
   let tunnel: { readonly id: string; readonly name: string } | null = null;
   return ManagedEndpointProvider.ManagedEndpointTunnelClient.of({
     get: (tunnelId) =>
@@ -134,6 +158,12 @@ function makePersistentTunnelClient(calls: TunnelCall[] = []) {
     putConfiguration: (tunnelId, tunnelConfig) =>
       Effect.sync(() => {
         calls.push({ operation: "putConfiguration", input: { tunnelId, tunnelConfig } });
+        configuration.current = tunnelConfig;
+      }),
+    getConfiguration: (tunnelId: string) =>
+      Effect.sync(() => {
+        calls.push({ operation: "getConfiguration", input: tunnelId });
+        return configuration.current ?? { ingress: [] };
       }),
     getToken: (tunnelId) =>
       Effect.sync(() => {
@@ -787,6 +817,44 @@ describe("ManagedEndpointProvider", () => {
     });
   });
 
+  it.effect("reads a tunnel without remote configuration as having no ingress", () => {
+    const tunnelClient = {
+      getConfiguration: () =>
+        Effect.fail(
+          new ZeroTrust.TunnelConfigurationNotFound({
+            code: 1055,
+            message: "Tunnel configuration not found",
+          }),
+        ),
+    } as unknown as Cloudflare.Tunnel.ReadWriteTunnelClient;
+    const layer = ManagedEndpointProvider.layerCloudflareBindings(
+      tunnelClient,
+      {} as Cloudflare.DNS.ReadWriteDnsClient,
+      {} as Alchemy.BaseRuntimeContext,
+    ).pipe(
+      Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)),
+      Layer.provide(RelayConfiguration.layer(config)),
+      Layer.provide(
+        Layer.succeed(ManagedEndpointAllocations.ManagedEndpointAllocations, makeAllocations()),
+      ),
+      Layer.provide(Layer.succeed(ManagedTunnelLimits.ManagedTunnelLimits, makeTunnelLimits())),
+      Layer.provide(
+        Layer.succeed(
+          CloudflareApiPause.CloudflareApiPause,
+          CloudflareApiPause.CloudflareApiPause.of({
+            pausedUntilMillis: Effect.succeed(0),
+            pauseUntil: () => Effect.void,
+          }),
+        ),
+      ),
+    );
+
+    return Effect.gen(function* () {
+      const tunnels = yield* ManagedEndpointProvider.ManagedEndpointTunnelClient;
+      expect(yield* tunnels.getConfiguration("tunnel-id")).toEqual({ ingress: [] });
+    }).pipe(Effect.provide(layer));
+  });
+
   it.effect("does not hide non-not-found checkpoint update failures", () => {
     const dnsCalls: DnsCall[] = [];
     const failure = new ManagedEndpointProvider.ManagedEndpointDnsClientError({
@@ -1139,6 +1207,72 @@ describe("ManagedEndpointProvider", () => {
     }).pipe(Effect.provide(layer));
   });
 
+  it.effect("restores Cloudflare ingress that drifted from the recorded origin", () => {
+    const tunnelCalls: TunnelCall[] = [];
+    const configuration: { current: TunnelConfiguration | null } = { current: null };
+    const layer = layerProvider(makePersistentTunnelClient(tunnelCalls, configuration));
+
+    return Effect.gen(function* () {
+      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
+      const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
+      const origin = { localHttpHost: "127.0.0.1", localHttpPort: 3773 } as const;
+      const provisioned = yield* provider.provision({ ...key, origin });
+      const reconcile = provider.reconcileOrigin({
+        ...key,
+        tunnelId: provisioned.runtime.tunnelId!,
+        origin,
+        endpoint: provisioned.endpoint,
+      });
+      tunnelCalls.length = 0;
+
+      expect(yield* reconcile).toBe("ready");
+      expect(tunnelCalls.map((call) => call.operation)).toEqual(["get", "getConfiguration"]);
+
+      configuration.current = {
+        ingress: [
+          { hostname: expectedManagedHostname("env_ABC"), service: "http://127.0.0.1:4711" },
+          { service: "http_status:404" },
+        ],
+      };
+      expect(yield* reconcile).toBe("ready");
+      expect(configuration.current).toEqual({
+        ingress: [
+          { hostname: expectedManagedHostname("env_ABC"), service: "http://127.0.0.1:3773" },
+          { service: "http_status:404" },
+        ],
+      });
+
+      configuration.current = {
+        ingress: [
+          { hostname: "*.t3code.test", service: "http://127.0.0.1:4711" },
+          { hostname: expectedManagedHostname("env_ABC"), service: "http://127.0.0.1:3773" },
+          { service: "http_status:404" },
+        ],
+      };
+      expect(yield* reconcile).toBe("ready");
+      expect(configuration.current.ingress[0]).toEqual({
+        hostname: expectedManagedHostname("env_ABC"),
+        service: "http://127.0.0.1:3773",
+      });
+
+      configuration.current = {
+        ingress: [
+          {
+            hostname: expectedManagedHostname("env_ABC"),
+            path: "/api",
+            service: "http://127.0.0.1:3773",
+          },
+          { service: "http_status:404" },
+        ],
+      };
+      expect(yield* reconcile).toBe("ready");
+      expect(configuration.current.ingress[0]).toEqual({
+        hostname: expectedManagedHostname("env_ABC"),
+        service: "http://127.0.0.1:3773",
+      });
+    }).pipe(Effect.provide(layer));
+  });
+
   it.effect("rejects an active endpoint that does not match the allocation hostname", () => {
     const layer = layerProvider(makePersistentTunnelClient());
 

infra/relay/src/environments/ManagedEndpointProvider.ts

@@ -216,6 +216,7 @@ const ManagedEndpointTunnelClientOperation = Schema.Literals([
   "list",
   "create",
   "put-configuration",
+  "get-configuration",
   "get-token",
   "delete",
 ]);
@@ -265,6 +266,16 @@ export class ManagedEndpointTunnelClient extends Context.Service<
         }>;
       },
     ) => Effect.Effect<unknown, ManagedEndpointTunnelClientError>;
+    readonly getConfiguration: (tunnelId: string) => Effect.Effect<
+      {
+        readonly ingress: ReadonlyArray<{
+          readonly hostname?: string | null;
+          readonly path?: string | null;
+          readonly service: string;
+        }>;
+      },
+      ManagedEndpointTunnelClientError
+    >;
     readonly getToken: (
       tunnelId: string,
     ) => Effect.Effect<string, ManagedEndpointTunnelClientError>;
@@ -756,32 +767,53 @@ const make = Effect.gen(function* () {
           hostname: allocation.hostname,
         });
       }
+      const verifyTunnel = <A>(read: Effect.Effect<A, ManagedEndpointTunnelClientError>) =>
+        read.pipe(
+          Effect.asSome,
+          Effect.catchTags({
+            ManagedEndpointTunnelClientError: (cause) =>
+              isManagedEndpointNotFound(cause.cause)
+                ? Effect.succeedNone
+                : Effect.fail(
+                    new ManagedEndpointProvisioningFailed({
+                      userId: input.userId,
+                      environmentId: input.environmentId,
+                      stage: "verify-tunnel",
+                      tunnelId: input.tunnelId,
+                      cause,
+                    }),
+                  ),
+          }),
+        );
       // Released records retain tunnel ids; verify Cloudflare still has the tunnel before reconnecting.
-      const recorded = yield* tunnels.get(input.tunnelId).pipe(
-        Effect.asSome,
-        Effect.catchTags({
-          ManagedEndpointTunnelClientError: (cause) =>
-            isManagedEndpointNotFound(cause.cause)
-              ? Effect.succeedNone
-              : Effect.fail(
-                  new ManagedEndpointProvisioningFailed({
-                    userId: input.userId,
-                    environmentId: input.environmentId,
-                    stage: "verify-tunnel",
-                    tunnelId: input.tunnelId,
-                    cause,
-                  }),
-                ),
-        }),
-      );
+      const recorded = yield* verifyTunnel(tunnels.get(input.tunnelId));
       if (Option.isNone(recorded) || recorded.value.deletedAt) {
         return "recovery_required";
       }
+      const originService = formatOriginService(input.origin);
       if (
         allocation.origin?.localHttpHost === input.origin.localHttpHost &&
         allocation.origin.localHttpPort === input.origin.localHttpPort
       ) {
-        return "ready";
+        // Live ingress can drift from the recorded origin; only a matching rule skips the rewrite.
+        const configuration = yield* verifyTunnel(tunnels.getConfiguration(input.tunnelId));
+        if (Option.isNone(configuration)) {
+          return "recovery_required";
+        }
+        const hostname = normalizeHostname(allocation.hostname);
+        // Cloudflare routes by the first matching rule, so an earlier wildcard or catch-all wins.
+        const rule = configuration.value.ingress.find((entry) => {
+          if (entry.hostname == null) return true;
+          const candidate = normalizeHostname(entry.hostname);
+          return (
+            candidate === hostname ||
+            (candidate.startsWith("*.") && hostname.endsWith(candidate.slice(1)))
+          );
+        });
+        // A path-restricted rule leaves other paths to later rules, so it cannot prove routing.
+        if (rule?.service === originService && !rule.path) {
+          return "ready";
+        }
       }
 
       const updated = yield* allocations
@@ -795,10 +827,7 @@ const make = Effect.gen(function* () {
           tunnels
             .putConfiguration(input.tunnelId, {
               ingress: [
-                {
-                  hostname: allocation.hostname,
-                  service: formatOriginService(input.origin),
-                },
+                { hostname: allocation.hostname, service: originService },
                 { service: "http_status:404" },
               ],
             })
@@ -1606,6 +1635,22 @@ export const layerCloudflareBindings = (
                       }),
                   ),
                 ),
+              getConfiguration: (tunnelId) =>
+                gate(tunnelClient.getConfiguration(tunnelId)).pipe(
+                  Effect.map(({ config }) => ({ ingress: config?.ingress ?? [] })),
+                  // Cloudflare reports a tunnel without remote configuration as code 1055, not 404.
+                  Effect.catchTags({
+                    TunnelConfigurationNotFound: () => Effect.succeed({ ingress: [] }),
+                  }),
+                  Effect.mapError(
+                    (cause) =>
+                      new ManagedEndpointTunnelClientError({
+                        operation: "get-configuration",
+                        tunnelId,
+                        cause,
+                      }),
+                  ),
+                ),
               getToken: (tunnelId) =>
                 gate(tunnelClient.getToken(tunnelId)).pipe(
                   Effect.mapError(

infra/relay/src/environments/ManagedEndpointReaper.test.ts

@@ -140,6 +140,7 @@ function harness(input?: {
       }),
     create: () => Effect.die("unused"),
     putConfiguration: () => Effect.die("unused"),
+    getConfiguration: () => Effect.die("unused"),
     getToken: () => Effect.die("unused"),
     delete: (tunnelId) =>
       input?.failAllDeletes === true ||