diff --git a/sdk/keyvault/azure-security-keyvault-certificates/src/test/java/com/azure/security/keyvault/certificates/CertificateClientTestBase.java b/sdk/keyvault/azure-security-keyvault-certificates/src/test/java/com/azure/security/keyvault/certificates/CertificateClientTestBase.java
index 3a97fcc9eb77..4ddb143dc68a 100644
--- a/sdk/keyvault/azure-security-keyvault-certificates/src/test/java/com/azure/security/keyvault/certificates/CertificateClientTestBase.java
+++ b/sdk/keyvault/azure-security-keyvault-certificates/src/test/java/com/azure/security/keyvault/certificates/CertificateClientTestBase.java
@@ -8,6 +8,7 @@
 import com.azure.core.http.HttpClient;
 import com.azure.core.http.HttpPipeline;
 import com.azure.core.http.HttpPipelineBuilder;
+import com.azure.core.http.policy.ExponentialBackoff;
 import com.azure.core.http.policy.HttpPipelinePolicy;
 import com.azure.core.http.policy.HttpPolicyProviders;
 import com.azure.core.http.policy.RetryPolicy;
@@ -15,6 +16,7 @@
 import com.azure.core.http.policy.HttpLoggingPolicy;
 import com.azure.core.http.policy.HttpLogOptions;
 import com.azure.core.http.policy.HttpLogDetailLevel;
+import com.azure.core.http.policy.RetryStrategy;
 import com.azure.core.http.policy.UserAgentPolicy;
 import com.azure.core.test.TestBase;
 import com.azure.core.util.Configuration;
@@ -36,6 +38,7 @@
 import com.azure.security.keyvault.certificates.models.WellKnownIssuerNames;
 import java.io.BufferedReader;
 import java.io.FileReader;
+import java.time.Duration;
 import java.util.stream.Stream;
 import org.junit.jupiter.api.Test;
 
@@ -92,7 +95,8 @@ HttpPipeline getHttpPipeline(HttpClient httpClient, CertificateServiceVersion se
         policies.add(new UserAgentPolicy(SDK_NAME, SDK_VERSION,
             Configuration.getGlobalConfiguration().clone(), serviceVersion));
         HttpPolicyProviders.addBeforeRetryPolicies(policies);
-        policies.add(new RetryPolicy());
+        RetryStrategy strategy = new ExponentialBackoff(5, Duration.ofSeconds(2), Duration.ofSeconds(16));
+        policies.add(new RetryPolicy(strategy));
         if (credential != null) {
             policies.add(new BearerTokenAuthenticationPolicy(credential, CertificateAsyncClient.KEY_VAULT_SCOPE));
         }
diff --git a/sdk/keyvault/azure-security-keyvault-keys/src/test/java/com/azure/security/keyvault/keys/KeyClientTestBase.java b/sdk/keyvault/azure-security-keyvault-keys/src/test/java/com/azure/security/keyvault/keys/KeyClientTestBase.java
index accf53cb1014..223f20924ed2 100644
--- a/sdk/keyvault/azure-security-keyvault-keys/src/test/java/com/azure/security/keyvault/keys/KeyClientTestBase.java
+++ b/sdk/keyvault/azure-security-keyvault-keys/src/test/java/com/azure/security/keyvault/keys/KeyClientTestBase.java
@@ -9,12 +9,14 @@
 import com.azure.core.http.HttpPipeline;
 import com.azure.core.http.HttpPipelineBuilder;
 import com.azure.core.http.policy.BearerTokenAuthenticationPolicy;
+import com.azure.core.http.policy.ExponentialBackoff;
 import com.azure.core.http.policy.HttpLogDetailLevel;
 import com.azure.core.http.policy.HttpLogOptions;
 import com.azure.core.http.policy.HttpLoggingPolicy;
 import com.azure.core.http.policy.HttpPipelinePolicy;
 import com.azure.core.http.policy.HttpPolicyProviders;
 import com.azure.core.http.policy.RetryPolicy;
+import com.azure.core.http.policy.RetryStrategy;
 import com.azure.core.http.policy.UserAgentPolicy;
 import com.azure.core.http.rest.Response;
 import com.azure.core.test.TestBase;
@@ -24,6 +26,8 @@
 import com.azure.security.keyvault.keys.models.CreateKeyOptions;
 import com.azure.security.keyvault.keys.models.KeyType;
 import com.azure.security.keyvault.keys.models.KeyVaultKey;
+
+import java.time.Duration;
 import java.util.stream.Stream;
 import org.junit.jupiter.api.Test;
 
@@ -77,7 +81,8 @@ HttpPipeline getHttpPipeline(HttpClient httpClient, KeyServiceVersion serviceVer
         final List<HttpPipelinePolicy> policies = new ArrayList<>();
         policies.add(new UserAgentPolicy(SDK_NAME, SDK_VERSION,  Configuration.getGlobalConfiguration().clone(), serviceVersion));
         HttpPolicyProviders.addBeforeRetryPolicies(policies);
-        policies.add(new RetryPolicy());
+        RetryStrategy strategy = new ExponentialBackoff(5, Duration.ofSeconds(2), Duration.ofSeconds(16));
+        policies.add(new RetryPolicy(strategy));
         if (credential != null) {
             policies.add(new BearerTokenAuthenticationPolicy(credential, KeyAsyncClient.KEY_VAULT_SCOPE));
         }
diff --git a/sdk/keyvault/azure-security-keyvault-keys/src/test/java/com/azure/security/keyvault/keys/cryptography/CryptographyClientTestBase.java b/sdk/keyvault/azure-security-keyvault-keys/src/test/java/com/azure/security/keyvault/keys/cryptography/CryptographyClientTestBase.java
index 27e25ec7f633..14ff3154a708 100644
--- a/sdk/keyvault/azure-security-keyvault-keys/src/test/java/com/azure/security/keyvault/keys/cryptography/CryptographyClientTestBase.java
+++ b/sdk/keyvault/azure-security-keyvault-keys/src/test/java/com/azure/security/keyvault/keys/cryptography/CryptographyClientTestBase.java
@@ -9,12 +9,14 @@
 import com.azure.core.http.HttpPipeline;
 import com.azure.core.http.HttpPipelineBuilder;
 import com.azure.core.http.policy.BearerTokenAuthenticationPolicy;
+import com.azure.core.http.policy.ExponentialBackoff;
 import com.azure.core.http.policy.HttpLogDetailLevel;
 import com.azure.core.http.policy.HttpLogOptions;
 import com.azure.core.http.policy.HttpLoggingPolicy;
 import com.azure.core.http.policy.HttpPipelinePolicy;
 import com.azure.core.http.policy.HttpPolicyProviders;
 import com.azure.core.http.policy.RetryPolicy;
+import com.azure.core.http.policy.RetryStrategy;
 import com.azure.core.http.policy.UserAgentPolicy;
 import com.azure.core.test.TestBase;
 import com.azure.core.util.Configuration;
@@ -30,6 +32,7 @@
 import java.security.spec.KeySpec;
 import java.security.spec.RSAPrivateCrtKeySpec;
 import java.security.spec.RSAPublicKeySpec;
+import java.time.Duration;
 import java.util.*;
 import java.util.function.Consumer;
 
@@ -69,7 +72,8 @@ HttpPipeline getHttpPipeline(HttpClient httpClient, ServiceVersion serviceVersio
         final List<HttpPipelinePolicy> policies = new ArrayList<>();
         policies.add(new UserAgentPolicy(SDK_NAME, SDK_VERSION, Configuration.getGlobalConfiguration().clone(), serviceVersion));
         HttpPolicyProviders.addBeforeRetryPolicies(policies);
-        policies.add(new RetryPolicy());
+        RetryStrategy strategy = new ExponentialBackoff(5, Duration.ofSeconds(2), Duration.ofSeconds(16));
+        policies.add(new RetryPolicy(strategy));
         if (credential != null) {
             policies.add(new BearerTokenAuthenticationPolicy(credential, CryptographyAsyncClient.KEY_VAULT_SCOPE));
         }
diff --git a/sdk/keyvault/azure-security-keyvault-keys/src/test/java/com/azure/security/keyvault/keys/cryptography/KeyEncryptionKeyClientTestBase.java b/sdk/keyvault/azure-security-keyvault-keys/src/test/java/com/azure/security/keyvault/keys/cryptography/KeyEncryptionKeyClientTestBase.java
index 672761426912..53cd1817501c 100644
--- a/sdk/keyvault/azure-security-keyvault-keys/src/test/java/com/azure/security/keyvault/keys/cryptography/KeyEncryptionKeyClientTestBase.java
+++ b/sdk/keyvault/azure-security-keyvault-keys/src/test/java/com/azure/security/keyvault/keys/cryptography/KeyEncryptionKeyClientTestBase.java
@@ -9,18 +9,21 @@
 import com.azure.core.http.HttpPipeline;
 import com.azure.core.http.HttpPipelineBuilder;
 import com.azure.core.http.policy.BearerTokenAuthenticationPolicy;
+import com.azure.core.http.policy.ExponentialBackoff;
 import com.azure.core.http.policy.HttpLogDetailLevel;
 import com.azure.core.http.policy.HttpLogOptions;
 import com.azure.core.http.policy.HttpLoggingPolicy;
 import com.azure.core.http.policy.HttpPipelinePolicy;
 import com.azure.core.http.policy.HttpPolicyProviders;
 import com.azure.core.http.policy.RetryPolicy;
+import com.azure.core.http.policy.RetryStrategy;
 import com.azure.core.http.policy.UserAgentPolicy;
 import com.azure.core.test.TestBase;
 import com.azure.core.util.Configuration;
 import com.azure.identity.ClientSecretCredentialBuilder;
 import org.junit.jupiter.api.Test;
 
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
@@ -61,7 +64,8 @@ HttpPipeline getHttpPipeline(HttpClient httpClient, CryptographyServiceVersion s
         final List<HttpPipelinePolicy> policies = new ArrayList<>();
         policies.add(new UserAgentPolicy(SDK_NAME, SDK_VERSION,  Configuration.getGlobalConfiguration().clone(), serviceVersion));
         HttpPolicyProviders.addBeforeRetryPolicies(policies);
-        policies.add(new RetryPolicy());
+        RetryStrategy strategy = new ExponentialBackoff(5, Duration.ofSeconds(2), Duration.ofSeconds(16));
+        policies.add(new RetryPolicy(strategy));
         if (credential != null) {
             policies.add(new BearerTokenAuthenticationPolicy(credential, CryptographyAsyncClient.KEY_VAULT_SCOPE));
         }
diff --git a/sdk/keyvault/azure-security-keyvault-secrets/src/test/java/com/azure/security/keyvault/secrets/SecretClientTestBase.java b/sdk/keyvault/azure-security-keyvault-secrets/src/test/java/com/azure/security/keyvault/secrets/SecretClientTestBase.java
index f5872b4ffcc8..1180231b0967 100644
--- a/sdk/keyvault/azure-security-keyvault-secrets/src/test/java/com/azure/security/keyvault/secrets/SecretClientTestBase.java
+++ b/sdk/keyvault/azure-security-keyvault-secrets/src/test/java/com/azure/security/keyvault/secrets/SecretClientTestBase.java
@@ -9,12 +9,14 @@
 import com.azure.core.http.HttpPipeline;
 import com.azure.core.http.HttpPipelineBuilder;
 import com.azure.core.http.policy.BearerTokenAuthenticationPolicy;
+import com.azure.core.http.policy.ExponentialBackoff;
 import com.azure.core.http.policy.HttpLogDetailLevel;
 import com.azure.core.http.policy.HttpLogOptions;
 import com.azure.core.http.policy.HttpLoggingPolicy;
 import com.azure.core.http.policy.HttpPipelinePolicy;
 import com.azure.core.http.policy.HttpPolicyProviders;
 import com.azure.core.http.policy.RetryPolicy;
+import com.azure.core.http.policy.RetryStrategy;
 import com.azure.core.http.policy.UserAgentPolicy;
 import com.azure.core.http.rest.Response;
 import com.azure.core.test.TestBase;
@@ -23,6 +25,8 @@
 import com.azure.identity.ClientSecretCredentialBuilder;
 import com.azure.security.keyvault.secrets.models.KeyVaultSecret;
 import com.azure.security.keyvault.secrets.models.SecretProperties;
+
+import java.time.Duration;
 import java.util.stream.Stream;
 import org.junit.jupiter.api.Test;
 
@@ -77,7 +81,8 @@ HttpPipeline getHttpPipeline(HttpClient httpClient, SecretServiceVersion service
         final List<HttpPipelinePolicy> policies = new ArrayList<>();
         policies.add(new UserAgentPolicy(SDK_NAME, SDK_VERSION,  Configuration.getGlobalConfiguration().clone(), serviceVersion));
         HttpPolicyProviders.addBeforeRetryPolicies(policies);
-        policies.add(new RetryPolicy());
+        RetryStrategy strategy = new ExponentialBackoff(5, Duration.ofSeconds(2), Duration.ofSeconds(16));
+        policies.add(new RetryPolicy(strategy));
         if (credential != null) {
             policies.add(new BearerTokenAuthenticationPolicy(credential, SecretAsyncClient.KEY_VAULT_SCOPE));
         }
diff --git a/sdk/keyvault/tests.yml b/sdk/keyvault/tests.yml
index f2d81a07dd69..b9ad3be410ef 100644
--- a/sdk/keyvault/tests.yml
+++ b/sdk/keyvault/tests.yml
@@ -4,7 +4,7 @@ jobs:
   - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml
     parameters:
       TimeoutInMinutes: 240
-      MaxParallel: 2
+      MaxParallel: 6
       ServiceDirectory: keyvault
       Artifacts:
         - name: azure-security-keyvault-certificates