diff --git a/internal/gatewayapi/backendtrafficpolicy.go b/internal/gatewayapi/backendtrafficpolicy.go
index 7229f58adc..5f0330cce5 100644
--- a/internal/gatewayapi/backendtrafficpolicy.go
+++ b/internal/gatewayapi/backendtrafficpolicy.go
@@ -930,6 +930,18 @@ func buildResponseOverride(policy *egv1a1.BackendTrafficPolicy, resources *resou
 	}, nil
 }
 
+func checkResponseBodySize(b *string) error {
+	// Make this configurable in the future
+	// https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/route/v3/route.proto.html#max_direct_response_body_size_bytes
+	maxDirectResponseSize := 4096
+	lenB := len(*b)
+	if lenB > maxDirectResponseSize {
+		return fmt.Errorf("response.body size %d greater than the max size %d", lenB, maxDirectResponseSize)
+	}
+
+	return nil
+}
+
 func getCustomResponseBody(body *egv1a1.CustomResponseBody, resources *resource.Resources, policyNs string) (*string, error) {
 	if body != nil && body.Type != nil && *body.Type == egv1a1.ResponseValueTypeValueRef {
 		cm := resources.GetConfigMap(policyNs, string(body.ValueRef.Name))
@@ -937,12 +949,18 @@ func getCustomResponseBody(body *egv1a1.CustomResponseBody, resources *resource.
 			b, dataOk := cm.Data["response.body"]
 			switch {
 			case dataOk:
+				if err := checkResponseBodySize(&b); err != nil {
+					return nil, err
+				}
 				return &b, nil
 			case len(cm.Data) > 0: // Fallback to the first key if response.body is not found
 				for _, value := range cm.Data {
 					b = value
 					break
 				}
+				if err := checkResponseBodySize(&b); err != nil {
+					return nil, err
+				}
 				return &b, nil
 			default:
 				return nil, fmt.Errorf("can't find the key response.body in the referenced configmap %s", body.ValueRef.Name)
@@ -952,6 +970,9 @@ func getCustomResponseBody(body *egv1a1.CustomResponseBody, resources *resource.
 			return nil, fmt.Errorf("can't find the referenced configmap %s", body.ValueRef.Name)
 		}
 	} else if body != nil && body.Inline != nil {
+		if err := checkResponseBodySize(body.Inline); err != nil {
+			return nil, err
+		}
 		return body.Inline, nil
 	}
 
diff --git a/internal/gatewayapi/testdata/httproute-with-direct-response.in.yaml b/internal/gatewayapi/testdata/httproute-with-direct-response.in.yaml
index bd9a316227..29f5b5c7f3 100644
--- a/internal/gatewayapi/testdata/httproute-with-direct-response.in.yaml
+++ b/internal/gatewayapi/testdata/httproute-with-direct-response.in.yaml
@@ -49,7 +49,7 @@ httpRoutes:
 - apiVersion: gateway.networking.k8s.io/v1
   kind: HTTPRoute
   metadata:
-    name: direct-response-with-errors
+    name: direct-response-with-value-not-found
     namespace: default
   spec:
     parentRefs:
@@ -67,6 +67,27 @@ httpRoutes:
           group: gateway.envoyproxy.io
           kind: HTTPRouteFilter
           name: direct-response-value-ref-not-found
+- apiVersion: gateway.networking.k8s.io/v1
+  kind: HTTPRoute
+  metadata:
+    name: direct-response-too-long
+    namespace: default
+  spec:
+    parentRefs:
+    - name: gateway-1
+      namespace: envoy-gateway
+      sectionName: http
+    rules:
+    - matches:
+      - path:
+          type: PathPrefix
+          value: /too-long
+      filters:
+      - type: ExtensionRef
+        extensionRef:
+          group: gateway.envoyproxy.io
+          kind: HTTPRouteFilter
+          name: direct-response-too-long
 configMaps:
 - apiVersion: v1
   kind: ConfigMap
@@ -117,3 +138,14 @@ httpFilters:
           group: ""
           kind: ConfigMap
           name: value-ref-response
+- apiVersion: gateway.envoyproxy.io/v1alpha1
+  kind: HTTPRouteFilter
+  metadata:
+    name: direct-response-too-long
+    namespace: default
+  spec:
+    directResponse:
+      contentType: text/plain
+      body:
+        type: Inline
+        inline: "-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"
diff --git a/internal/gatewayapi/testdata/httproute-with-direct-response.out.yaml b/internal/gatewayapi/testdata/httproute-with-direct-response.out.yaml
index ca1005fdef..fbe18fd925 100644
--- a/internal/gatewayapi/testdata/httproute-with-direct-response.out.yaml
+++ b/internal/gatewayapi/testdata/httproute-with-direct-response.out.yaml
@@ -17,7 +17,7 @@ gateways:
       protocol: HTTP
   status:
     listeners:
-    - attachedRoutes: 2
+    - attachedRoutes: 3
       conditions:
       - lastTransitionTime: null
         message: Sending translated listener configuration to the data plane
@@ -95,7 +95,7 @@ httpRoutes:
   kind: HTTPRoute
   metadata:
     creationTimestamp: null
-    name: direct-response-with-errors
+    name: direct-response-with-value-not-found
     namespace: default
   spec:
     parentRefs:
@@ -131,6 +131,47 @@ httpRoutes:
         name: gateway-1
         namespace: envoy-gateway
         sectionName: http
+- apiVersion: gateway.networking.k8s.io/v1
+  kind: HTTPRoute
+  metadata:
+    creationTimestamp: null
+    name: direct-response-too-long
+    namespace: default
+  spec:
+    parentRefs:
+    - name: gateway-1
+      namespace: envoy-gateway
+      sectionName: http
+    rules:
+    - filters:
+      - extensionRef:
+          group: gateway.envoyproxy.io
+          kind: HTTPRouteFilter
+          name: direct-response-too-long
+        type: ExtensionRef
+      matches:
+      - path:
+          type: PathPrefix
+          value: /too-long
+  status:
+    parents:
+    - conditions:
+      - lastTransitionTime: null
+        message: 'Invalid filter HTTPRouteFilter: response.body size 4097 greater
+          than the max size 4096'
+        reason: UnsupportedValue
+        status: "False"
+        type: Accepted
+      - lastTransitionTime: null
+        message: Resolved all the Object references for the Route
+        reason: ResolvedRefs
+        status: "True"
+        type: ResolvedRefs
+      controllerName: gateway.envoyproxy.io/gatewayclass-controller
+      parentRef:
+        name: gateway-1
+        namespace: envoy-gateway
+        sectionName: http
 infraIR:
   envoy-gateway/gateway-1:
     proxy: