diff --git a/azurerm/internal/services/keyvault/internal.go b/azurerm/internal/services/keyvault/internal.go
index 113d127cc6c6..1517c71a69a3 100644
--- a/azurerm/internal/services/keyvault/internal.go
+++ b/azurerm/internal/services/keyvault/internal.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"log"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/Azure/go-autorest/autorest"
@@ -69,8 +70,18 @@ func deleteAndOptionallyPurge(ctx context.Context, description string, shouldPur
 	}
 
 	log.Printf("[DEBUG] Purging %s..", description)
-	if _, err := helper.PurgeNestedItem(ctx); err != nil {
-		return fmt.Errorf("purging %s: %+v", description, err)
+	err := resource.Retry(time.Until(timeout), func() *resource.RetryError {
+		_, err := helper.PurgeNestedItem(ctx)
+		if err == nil {
+			return nil
+		}
+		if strings.Contains(err.Error(), "is currently being deleted") {
+			return resource.RetryableError(fmt.Errorf("%s is currently being deleted, retrying", description))
+		}
+		return resource.NonRetryableError(fmt.Errorf("Error purging of %s : %+v", description, err))
+	})
+	if err != nil {
+		return err
 	}
 
 	log.Printf("[DEBUG] Waiting for %s to finish purging..", description)
diff --git a/azurerm/internal/services/keyvault/key_vault_certificate_resource_test.go b/azurerm/internal/services/keyvault/key_vault_certificate_resource_test.go
index 5bab3fb1ba5d..b99d919084e1 100644
--- a/azurerm/internal/services/keyvault/key_vault_certificate_resource_test.go
+++ b/azurerm/internal/services/keyvault/key_vault_certificate_resource_test.go
@@ -243,6 +243,29 @@ func TestAccKeyVaultCertificate_withExternalAccessPolicy(t *testing.T) {
 	})
 }
 
+func TestAccKeyVaultCertificate_purge(t *testing.T) {
+	data := acceptance.BuildTestData(t, "azurerm_key_vault_certificate", "test")
+	r := KeyVaultCertificateResource{}
+
+	data.ResourceTest(t, r, []resource.TestStep{
+		{
+			Config: r.basicGenerate(data),
+			Check: resource.ComposeTestCheckFunc(
+				check.That(data.ResourceName).ExistsInAzure(r),
+				check.That(data.ResourceName).Key("secret_id").Exists(),
+				check.That(data.ResourceName).Key("certificate_data").Exists(),
+				check.That(data.ResourceName).Key("certificate_data_base64").Exists(),
+				check.That(data.ResourceName).Key("thumbprint").Exists(),
+				check.That(data.ResourceName).Key("certificate_attribute.0.created").Exists(),
+			),
+		},
+		{
+			Config:  r.basicGenerate(data),
+			Destroy: true,
+		},
+	})
+}
+
 func (t KeyVaultCertificateResource) Exists(ctx context.Context, clients *clients.Client, state *terraform.InstanceState) (*bool, error) {
 	keyVaultsClient := clients.KeyVault
 	client := clients.KeyVault.ManagementClient
diff --git a/azurerm/internal/services/keyvault/key_vault_key_resource_test.go b/azurerm/internal/services/keyvault/key_vault_key_resource_test.go
index e3f72040fc45..b9a0f371cfa6 100644
--- a/azurerm/internal/services/keyvault/key_vault_key_resource_test.go
+++ b/azurerm/internal/services/keyvault/key_vault_key_resource_test.go
@@ -265,6 +265,24 @@ func TestAccKeyVaultKey_withExternalAccessPolicy(t *testing.T) {
 	})
 }
 
+func TestAccKeyVaultKey_purge(t *testing.T) {
+	data := acceptance.BuildTestData(t, "azurerm_key_vault_key", "test")
+	r := KeyVaultKeyResource{}
+
+	data.ResourceTest(t, r, []resource.TestStep{
+		{
+			Config: r.basicEC(data),
+			Check: resource.ComposeTestCheckFunc(
+				check.That(data.ResourceName).ExistsInAzure(r),
+			),
+		},
+		{
+			Config:  r.basicEC(data),
+			Destroy: true,
+		},
+	})
+}
+
 func (r KeyVaultKeyResource) Exists(ctx context.Context, clients *clients.Client, state *terraform.InstanceState) (*bool, error) {
 	client := clients.KeyVault.ManagementClient
 	keyVaultsClient := clients.KeyVault
diff --git a/azurerm/internal/services/keyvault/key_vault_secret_resource_test.go b/azurerm/internal/services/keyvault/key_vault_secret_resource_test.go
index 9f9ac203765e..a8d022b97ae9 100644
--- a/azurerm/internal/services/keyvault/key_vault_secret_resource_test.go
+++ b/azurerm/internal/services/keyvault/key_vault_secret_resource_test.go
@@ -198,6 +198,25 @@ func TestAccKeyVaultSecret_withExternalAccessPolicy(t *testing.T) {
 	})
 }
 
+func TestAccKeyVaultSecret_purge(t *testing.T) {
+	data := acceptance.BuildTestData(t, "azurerm_key_vault_secret", "test")
+	r := KeyVaultSecretResource{}
+
+	data.ResourceTest(t, r, []resource.TestStep{
+		{
+			Config: r.basic(data),
+			Check: resource.ComposeTestCheckFunc(
+				check.That(data.ResourceName).ExistsInAzure(r),
+				check.That(data.ResourceName).Key("value").HasValue("rick-and-morty"),
+			),
+		},
+		{
+			Config:  r.basic(data),
+			Destroy: true,
+		},
+	})
+}
+
 func (KeyVaultSecretResource) Exists(ctx context.Context, clients *clients.Client, state *terraform.InstanceState) (*bool, error) {
 	client := clients.KeyVault.ManagementClient
 	keyVaultsClient := clients.KeyVault