diff --git a/internal/xds/translator/testdata/in/xds-ir/backend-tls-skip-verify.yaml b/internal/xds/translator/testdata/in/xds-ir/backend-tls-skip-verify.yaml
index fc74c38f36..f446ca58e5 100644
--- a/internal/xds/translator/testdata/in/xds-ir/backend-tls-skip-verify.yaml
+++ b/internal/xds/translator/testdata/in/xds-ir/backend-tls-skip-verify.yaml
@@ -19,7 +19,3 @@ http:
               name: "first-route-dest/backend/0"
               tls:
                 insecureSkipVerify: true
-                useSystemTrustStore: true
-                CACertificate:
-                  name: policy-btls/default-ca
-                sni: example.com
diff --git a/internal/xds/translator/testdata/out/xds-ir/backend-tls-skip-verify.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/backend-tls-skip-verify.clusters.yaml
index 540ba4fd71..915a64ca87 100644
--- a/internal/xds/translator/testdata/out/xds-ir/backend-tls-skip-verify.clusters.yaml
+++ b/internal/xds/translator/testdata/out/xds-ir/backend-tls-skip-verify.clusters.yaml
@@ -30,5 +30,4 @@
       typedConfig:
         '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
         commonTlsContext: {}
-        sni: example.com
   type: EDS
diff --git a/internal/xds/translator/translator.go b/internal/xds/translator/translator.go
index 862d8d7133..d04b7d965c 100644
--- a/internal/xds/translator/translator.go
+++ b/internal/xds/translator/translator.go
@@ -1060,7 +1060,8 @@ func addXdsCluster(tCtx *types.ResourceVersionTable, args *xdsClusterArgs) error
 	preferLocal := ptr.Deref(args.loadBalancer, ir.LoadBalancer{}).PreferLocal
 	xdsEndpoints := buildXdsClusterLoadAssignment(args.name, args.settings, preferLocal)
 	for _, ds := range args.settings {
-		if ds.TLS != nil {
+		shouldValidateTLS := ds.TLS != nil && !ds.TLS.InsecureSkipVerify
+		if shouldValidateTLS {
 			// Create an SDS secret for the CA certificate - either with inline bytes or with a filesystem ref
 			secret := buildXdsUpstreamTLSCASecret(ds.TLS)
 			if err := tCtx.AddXdsResource(resourcev3.SecretType, secret); err != nil {
diff --git a/test/e2e/testdata/backend-tls.yaml b/test/e2e/testdata/backend-tls.yaml
index b95dbaa900..1f6f7d0637 100644
--- a/test/e2e/testdata/backend-tls.yaml
+++ b/test/e2e/testdata/backend-tls.yaml
@@ -153,7 +153,7 @@ spec:
 apiVersion: gateway.envoyproxy.io/v1alpha1
 kind: Backend
 metadata:
-  name: backend-insecure-tls-verify
+  name: backend-insecure-tls-verify-and-mismatch-ca
   namespace: gateway-conformance-infra
 spec:
   endpoints:
@@ -166,7 +166,7 @@ spec:
 apiVersion: gateway.networking.k8s.io/v1
 kind: HTTPRoute
 metadata:
-  name: http-with-backend-insecure-skip-verify
+  name: http-with-backend-insecure-skip-verify-and-mismatch-ca
   namespace: gateway-conformance-infra
 spec:
   parentRefs:
@@ -175,9 +175,40 @@ spec:
     - matches:
         - path:
             type: PathPrefix
-            value: /backend-tls-skip-verify
+            value: /backend-tls-skip-verify-and-mismatch-ca
       backendRefs:
-        - name: backend-insecure-tls-verify
+        - name: backend-insecure-tls-verify-and-mismatch-ca
+          group: gateway.envoyproxy.io
+          kind: Backend
+---
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: Backend
+metadata:
+  name: backend-insecure-tls-verify-without-backend-tls-policy
+  namespace: gateway-conformance-infra
+spec:
+  endpoints:
+    - fqdn:
+        hostname: tls-backend-2.gateway-conformance-infra.svc.cluster.local
+        port: 443
+  tls:
+    insecureSkipVerify: true
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: http-with-backend-insecure-skip-verify-without-backend-tls-policy
+  namespace: gateway-conformance-infra
+spec:
+  parentRefs:
+    - name: same-namespace
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /backend-tls-skip-verify-without-backend-tls-policy
+      backendRefs:
+        - name: backend-insecure-tls-verify-without-backend-tls-policy
           group: gateway.envoyproxy.io
           kind: Backend
 ---
diff --git a/test/e2e/tests/backend_tls.go b/test/e2e/tests/backend_tls.go
index bb76ed9ef3..e711f79492 100644
--- a/test/e2e/tests/backend_tls.go
+++ b/test/e2e/tests/backend_tls.go
@@ -88,12 +88,29 @@ var BackendTLSTest = suite.ConformanceTest{
 		})
 
 		t.Run("with CA mismatch and skip tls verify", func(t *testing.T) {
-			routeNN := types.NamespacedName{Name: "http-with-backend-insecure-skip-verify", Namespace: ConformanceInfraNamespace}
+			routeNN := types.NamespacedName{Name: "http-with-backend-insecure-skip-verify-and-mismatch-ca", Namespace: ConformanceInfraNamespace}
 			gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN)
 
 			expectedResponse := http.ExpectedResponse{
 				Request: http.Request{
-					Path: "/backend-tls-skip-verify",
+					Path: "/backend-tls-skip-verify-and-mismatch-ca",
+				},
+				Response: http.Response{
+					StatusCode: 200, // Bad Request: Client sent an HTTP request to an HTTPS server
+				},
+				Namespace: ConformanceInfraNamespace,
+			}
+
+			http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, expectedResponse)
+		})
+
+		t.Run("without BackendTLSPolicy and skip tls verify", func(t *testing.T) {
+			routeNN := types.NamespacedName{Name: "http-with-backend-insecure-skip-verify-without-backend-tls-policy", Namespace: ConformanceInfraNamespace}
+			gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN)
+
+			expectedResponse := http.ExpectedResponse{
+				Request: http.Request{
+					Path: "/backend-tls-skip-verify-without-backend-tls-policy",
 				},
 				Response: http.Response{
 					StatusCode: 200, // Bad Request: Client sent an HTTP request to an HTTPS server