diff --git a/internal/xds/translator/route.go b/internal/xds/translator/route.go
index 0c3a47e38f..fd5c64648e 100644
--- a/internal/xds/translator/route.go
+++ b/internal/xds/translator/route.go
@@ -74,7 +74,7 @@ func buildXdsRoute(httpRoute *ir.HTTPRoute, httpListener *ir.HTTPListener) (*rou
 		router.Action = &routev3.Route_Redirect{Redirect: buildXdsRedirectAction(httpRoute)}
 	case httpRoute.URLRewrite != nil:
 		routeAction := buildXdsURLRewriteAction(httpRoute, httpRoute.URLRewrite, httpRoute.PathMatch)
-		routeAction.IdleTimeout = idleTimeout(httpRoute)
+		routeAction.IdleTimeout = idleTimeout(httpRoute, httpListener)
 		if httpRoute.Mirrors != nil {
 			routeAction.RequestMirrorPolicies = buildXdsRequestMirrorPolicies(httpRoute.Mirrors)
 		}
@@ -86,7 +86,7 @@ func buildXdsRoute(httpRoute *ir.HTTPRoute, httpListener *ir.HTTPListener) (*rou
 		router.Action = &routev3.Route_Route{Route: routeAction}
 	default:
 		routeAction := buildXdsRouteAction(httpRoute)
-		routeAction.IdleTimeout = idleTimeout(httpRoute)
+		routeAction.IdleTimeout = idleTimeout(httpRoute, httpListener)
 
 		if httpRoute.Mirrors != nil {
 			routeAction.RequestMirrorPolicies = buildXdsRequestMirrorPolicies(httpRoute.Mirrors)
@@ -408,21 +408,31 @@ func getEffectiveRequestTimeout(httpRoute *ir.HTTPRoute) *metav1.Duration {
 	return nil
 }
 
-func idleTimeout(httpRoute *ir.HTTPRoute) *durationpb.Duration {
-	rt := getEffectiveRequestTimeout(httpRoute)
-	timeout := time.Hour // Default to 1 hour
-	if rt != nil {
-		// Ensure is not less than the request timeout
-		if timeout < rt.Duration {
-			timeout = rt.Duration
-		}
+func idleTimeout(httpRoute *ir.HTTPRoute, httpListener *ir.HTTPListener) *durationpb.Duration {
+	// When a user-configured stream idle timeout exists at the listener level, avoid overriding it at the route level
+	// and allow the user-configured listener-level timeout to take effect.
+	// TODO: we may need to support route-level idle timeout in the BackendTrafficPolicy
+	if httpListener != nil &&
+		httpListener.Timeout != nil &&
+		httpListener.Timeout.HTTP != nil &&
+		httpListener.Timeout.HTTP.StreamIdleTimeout != nil {
+		return nil
+	}
 
+	// When a user-configured request timeout exists at the route level, and no user-configured stream idle timeout exists
+	// at the listener level, set a route-level idle timeout to avoid stream timeout before request timeout.
+	requestTimeout := getEffectiveRequestTimeout(httpRoute)
+	idleTimeout := time.Hour // Default to 1 hour
+	if requestTimeout != nil {
+		// Ensure the idle timeout is not less than the request timeout
+		if idleTimeout < requestTimeout.Duration {
+			idleTimeout = requestTimeout.Duration
+		}
 		// Disable idle timeout when request timeout is disabled
-		if rt.Duration == 0 {
-			timeout = 0
+		if requestTimeout.Duration == 0 {
+			idleTimeout = 0
 		}
-
-		return durationpb.New(timeout)
+		return durationpb.New(idleTimeout)
 	}
 	return nil
 }
diff --git a/internal/xds/translator/testdata/in/xds-ir/http-route-timeout.yaml b/internal/xds/translator/testdata/in/xds-ir/http-route-timeout.yaml
index 9164b6b082..64b4e12779 100644
--- a/internal/xds/translator/testdata/in/xds-ir/http-route-timeout.yaml
+++ b/internal/xds/translator/testdata/in/xds-ir/http-route-timeout.yaml
@@ -101,3 +101,32 @@ http:
         - host: "1.2.3.4"
           port: 50002
         name: "seventh-route-dest/backend/0"
+- name: "second-listener"
+  address: "::"
+  port: 10081
+  hostnames:
+  - "*"
+  path:
+    mergeSlashes: true
+    escapedSlashesAction: UnescapeAndRedirect
+  timeout:
+    http:
+      streamIdleTimeout: 10s
+  routes:
+  - name: "first-route"
+    hostname: "*"
+    traffic:
+      timeout:
+        http:
+          requestTimeout: 5s          # should not set route-level idle timeout because listener has it
+    headerMatches:
+    - name: user
+      stringMatch:
+      exact: "jason"
+    destination:
+      name: "first-route-dest"
+      settings:
+      - endpoints:
+        - host: "1.2.3.4"
+          port: 50000
+        name: "first-route-dest/backend/0"
diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-timeout.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-timeout.listeners.yaml
index 5dd5e46e3c..c2a422e285 100644
--- a/internal/xds/translator/testdata/out/xds-ir/http-route-timeout.listeners.yaml
+++ b/internal/xds/translator/testdata/out/xds-ir/http-route-timeout.listeners.yaml
@@ -33,3 +33,39 @@
   maxConnectionsToAcceptPerSocketEvent: 1
   name: first-listener
   perConnectionBufferLimitBytes: 32768
+- address:
+    socketAddress:
+      address: '::'
+      portValue: 10081
+  defaultFilterChain:
+    filters:
+    - name: envoy.filters.network.http_connection_manager
+      typedConfig:
+        '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+        commonHttpProtocolOptions:
+          headersWithUnderscoresAction: REJECT_REQUEST
+        http2ProtocolOptions:
+          initialConnectionWindowSize: 1048576
+          initialStreamWindowSize: 65536
+          maxConcurrentStreams: 100
+        httpFilters:
+        - name: envoy.filters.http.router
+          typedConfig:
+            '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+            suppressEnvoyHeaders: true
+        mergeSlashes: true
+        normalizePath: true
+        pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT
+        rds:
+          configSource:
+            ads: {}
+            resourceApiVersion: V3
+          routeConfigName: second-listener
+        serverHeaderTransformation: PASS_THROUGH
+        statPrefix: http-10081
+        streamIdleTimeout: 10s
+        useRemoteAddress: true
+    name: second-listener
+  maxConnectionsToAcceptPerSocketEvent: 1
+  name: second-listener
+  perConnectionBufferLimitBytes: 32768
diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-timeout.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-timeout.routes.yaml
index 4585dc7b0b..a1163a0428 100644
--- a/internal/xds/translator/testdata/out/xds-ir/http-route-timeout.routes.yaml
+++ b/internal/xds/translator/testdata/out/xds-ir/http-route-timeout.routes.yaml
@@ -76,3 +76,22 @@
         timeout: 0s
         upgradeConfigs:
         - upgradeType: websocket
+- ignorePortInHostMatching: true
+  name: second-listener
+  virtualHosts:
+  - domains:
+    - '*'
+    name: second-listener/*
+    routes:
+    - match:
+        headers:
+        - name: user
+          stringMatch:
+            exact: jason
+        prefix: /
+      name: first-route
+      route:
+        cluster: first-route-dest
+        timeout: 5s
+        upgradeConfigs:
+        - upgradeType: websocket