performance improvements on mount fetch

CHANGELOG.md

@@ -10,6 +10,9 @@ FEATURES:
 * Add support for `skip_import_rotation` and `skip_static_role_import_rotation` in `ldap_secret_backend_static_role` and `ldap_secret_backend` respectively. Requires Vault 1.16+ ([#2128](https://github.com/hashicorp/terraform-provider-vault/pull/2128)).
 * Improve logging to track full API exchanges between the provider and Vault ([#2139](https://github.com/hashicorp/terraform-provider-vault/pull/2139))
 
+IMPROVEMENTS:
+* Improve performance of READ operations across many resources: ([#2145](https://github.com/hashicorp/terraform-provider-vault/pull/2145)), ([#2152](https://github.com/hashicorp/terraform-provider-vault/pull/2152))
+
 ## 3.25.0 (Feb 14, 2024)
 
 FEATURES:

util/mountutil/mountutil.go

@@ -0,0 +1,81 @@
+package mountutil
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/hashicorp/terraform-provider-vault/internal/consts"
+	"github.com/hashicorp/vault/api"
+)
+
+// Error strings that are returned by the Vault API.
+const (
+	ErrVaultSecretMountNotFound = "No secret engine mount at"
+	ErrVaultAuthMountNotFound   = "No auth engine at"
+)
+
+// Error strings that are used internally by TFVP
+var (
+	// ErrMountNotFound is used to signal to resources that a secret or auth
+	// mount does not exist and should be removed from state.
+	ErrMountNotFound = errors.New("mount not found")
+)
+
+// GetMount will fetch the secret mount at the given path.
+func GetMount(ctx context.Context, client *api.Client, path string) (*api.MountOutput, error) {
+	mount, err := client.Sys().GetMountWithContext(ctx, path)
+	// Hardcoding the error string check is not ideal, but Vault does not
+	// return 404 in this case
+	if err != nil && strings.Contains(err.Error(), ErrVaultSecretMountNotFound) || mount == nil {
+		return nil, fmt.Errorf("%w: %s", ErrMountNotFound, err)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("error reading from Vault: %s", err)
+	}
+	return mount, nil
+}
+
+// GetAuthMount will fetch the auth mount at the given path.
+func GetAuthMount(ctx context.Context, client *api.Client, path string) (*api.MountOutput, error) {
+	mount, err := client.Sys().GetAuthWithContext(ctx, path)
+	// Hardcoding the error string check is not ideal, but Vault does not
+	// return 404 in this case
+	if err != nil && strings.Contains(err.Error(), ErrVaultAuthMountNotFound) || mount == nil {
+		return nil, fmt.Errorf("%w: %s", ErrMountNotFound, err)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("error reading from Vault: %s", err)
+	}
+	return mount, nil
+}
+
+// NormalizeMountPath to be in a form valid for accessing values from api.MountOutput
+func NormalizeMountPath(path string) string {
+	return TrimSlashes(path) + consts.PathDelim
+}
+
+// TrimSlashes from path.
+func TrimSlashes(path string) string {
+	return strings.Trim(path, consts.PathDelim)
+}
+
+// CheckMountEnabledWithContext in Vault
+func CheckMountEnabledWithContext(ctx context.Context, client *api.Client, path string) (bool, error) {
+	_, err := GetMount(ctx, client, path)
+	if errors.Is(err, ErrMountNotFound) {
+		return false, err
+	}
+
+	if err != nil {
+		return false, err
+	}
+
+	return true, nil
+}
+
+// CheckMountEnabled in Vault
+func CheckMountEnabled(client *api.Client, path string) (bool, error) {
+	return CheckMountEnabledWithContext(context.Background(), client, path)
+}

util/util.go

@@ -16,8 +16,6 @@ import (
 	"github.com/cenkalti/backoff/v4"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 	"github.com/hashicorp/vault/api"
-
-	"github.com/hashicorp/terraform-provider-vault/internal/consts"
 )
 
 type (
@@ -281,28 +279,6 @@ func SetResourceData(d *schema.ResourceData, data map[string]interface{}) error
 	return nil
 }
 
-// NormalizeMountPath to be in a form valid for accessing values from api.MountOutput
-func NormalizeMountPath(path string) string {
-	return TrimSlashes(path) + consts.PathDelim
-}
-
-// TrimSlashes from path.
-func TrimSlashes(path string) string {
-	return strings.Trim(path, consts.PathDelim)
-}
-
-// CheckMountEnabled in Vault, path must contain a trailing '/',
-func CheckMountEnabled(client *api.Client, path string) (bool, error) {
-	mounts, err := client.Sys().ListMounts()
-	if err != nil {
-		return false, err
-	}
-
-	_, ok := mounts[NormalizeMountPath(path)]
-
-	return ok, nil
-}
-
 // GetAPIRequestDataWithMap to pass to Vault from schema.ResourceData.
 // The fieldMap specifies the schema field to its vault constituent.
 // If the vault field is empty, then two fields are mapped 1:1.

vault/auth_mount.go

@@ -6,7 +6,6 @@ package vault
 import (
 	"fmt"
 	"log"
-	"strings"
 
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
@@ -77,19 +76,6 @@ func authMountTuneSchema() *schema.Schema {
 	}
 }
 
-func authMountInfoGet(client *api.Client, path string) (*api.AuthMount, error) {
-	auths, err := client.Sys().ListAuth()
-	if err != nil {
-		return nil, fmt.Errorf("error reading from auth mounts: %s", err)
-	}
-
-	authMount := auths[strings.Trim(path, "/")+"/"]
-	if authMount == nil {
-		return nil, fmt.Errorf("auth mount %s not present", path)
-	}
-	return authMount, nil
-}
-
 func authMountTune(client *api.Client, path string, configured interface{}) error {
 	input := expandAuthMethodTune(configured.(*schema.Set).List())
 
@@ -124,15 +110,3 @@ func authMountDisable(client *api.Client, path string) error {
 
 	return nil
 }
-
-// getAuthMountIfPresent will fetch the auth mount at the given path.
-func getAuthMountIfPresent(client *api.Client, path string) (*api.AuthMount, error) {
-	auth, err := client.Sys().GetAuth(path)
-	if err != nil {
-		return nil, fmt.Errorf("error reading from Vault: %s", err)
-	}
-	if auth.Accessor == "" {
-		return nil, fmt.Errorf("mount not found: %s", err)
-	}
-	return auth, nil
-}

vault/data_source_auth_backend.go

@@ -4,12 +4,14 @@
 package vault
 
 import (
+	"context"
 	"fmt"
 	"strings"
 
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 
 	"github.com/hashicorp/terraform-provider-vault/internal/provider"
+	"github.com/hashicorp/terraform-provider-vault/util/mountutil"
 )
 
 func authBackendDataSource() *schema.Resource {
@@ -66,29 +68,37 @@ func authBackendDataSourceRead(d *schema.ResourceData, meta interface{}) error {
 		return e
 	}
 
-	targetPath := d.Get("path").(string)
+	path := d.Get("path").(string)
 
-	auths, err := client.Sys().ListAuth()
+	auth, err := mountutil.GetAuthMount(context.Background(), client, path)
 	if err != nil {
 		return fmt.Errorf("error reading from Vault: %s", err)
 	}
 
-	for path, auth := range auths {
-		path = strings.TrimSuffix(path, "/")
-		if path == targetPath {
-			// Compatibility with resource_auth_backend id
-			d.SetId(path)
-			d.Set("type", auth.Type)
-			d.Set("description", auth.Description)
-			d.Set("accessor", auth.Accessor)
-			d.Set("default_lease_ttl_seconds", auth.Config.DefaultLeaseTTL)
-			d.Set("max_lease_ttl_seconds", auth.Config.MaxLeaseTTL)
-			d.Set("listing_visibility", auth.Config.ListingVisibility)
-			d.Set("local", auth.Local)
-			return nil
-		}
+	path = strings.TrimSuffix(path, "/")
+	d.SetId(path)
+
+	if err := d.Set("type", auth.Type); err != nil {
+		return err
+	}
+	if err := d.Set("description", auth.Description); err != nil {
+		return err
+	}
+	if err := d.Set("accessor", auth.Accessor); err != nil {
+		return err
+	}
+	if err := d.Set("default_lease_ttl_seconds", auth.Config.DefaultLeaseTTL); err != nil {
+		return err
+	}
+	if err := d.Set("max_lease_ttl_seconds", auth.Config.MaxLeaseTTL); err != nil {
+		return err
+	}
+	if err := d.Set("listing_visibility", auth.Config.ListingVisibility); err != nil {
+		return err
+	}
+	if err := d.Set("local", auth.Local); err != nil {
+		return err
 	}
 
-	// If we fell out here then we didn't find our Auth in the list.
 	return nil
 }

vault/okta.go

@@ -50,7 +50,6 @@ func listOktaUsers(client *api.Client, path string) ([]string, error) {
 
 func readOktaUser(client *api.Client, path string, username string) (*oktaUser, error) {
 	secret, err := client.Logical().Read(oktaUserEndpoint(path, username))
-
 	if err != nil {
 		return nil, err
 	}
@@ -76,24 +75,6 @@ func deleteOktaUser(client *api.Client, path, username string) error {
 	return err
 }
 
-func isOktaAuthBackendPresent(client *api.Client, path string) (bool, error) {
-	auths, err := client.Sys().ListAuth()
-	if err != nil {
-		return false, fmt.Errorf("error reading from Vault: %s", err)
-	}
-
-	configuredPath := path + "/"
-
-	for authBackendPath, auth := range auths {
-
-		if auth.Type == "okta" && authBackendPath == configuredPath {
-			return true, nil
-		}
-	}
-
-	return false, nil
-}
-
 func isOktaGroupPresent(client *api.Client, path, name string) (bool, error) {
 	secret, err := client.Logical().Read(oktaGroupEndpoint(path, name))
 	if err != nil {
@@ -122,7 +103,6 @@ func listOktaGroups(client *api.Client, path string) ([]string, error) {
 
 func readOktaGroup(client *api.Client, path string, name string) (*oktaGroup, error) {
 	secret, err := client.Logical().Read(oktaGroupEndpoint(path, name))
-
 	if err != nil {
 		return nil, err
 	}

vault/resource_ad_secret_backend.go

@@ -4,13 +4,16 @@
 package vault
 
 import (
+	"context"
+	"errors"
 	"fmt"
 	"log"
 	"strings"
 
 	"github.com/hashicorp/terraform-provider-vault/internal/consts"
 	"github.com/hashicorp/terraform-provider-vault/internal/provider"
 	"github.com/hashicorp/terraform-provider-vault/util"
+	"github.com/hashicorp/terraform-provider-vault/util/mountutil"
 
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 	"github.com/hashicorp/vault/api"
@@ -362,19 +365,21 @@
 	path := d.Id()
 	log.Printf("[DEBUG] Reading %q", path)
 
-	mountResp, err := client.Sys().MountConfig(path)
-	if err != nil && util.Is404(err) {
-		log.Printf("[WARN] %q not found, removing from state", path)
+	mount, err := mountutil.GetMount(context.Background(), client, path)
+	if errors.Is(err, mountutil.ErrMountNotFound) {
+		log.Printf("[WARN] Mount %q not found, removing from state.", path)
 		d.SetId("")
 		return nil
-	} else if err != nil {
-		return fmt.Errorf("error reading %q: %s", path, err)
+	}
+
+	if err != nil {
+		return err
 	}
 
 	d.Set("backend", d.Id())
 
-	d.Set("default_lease_ttl_seconds", mountResp.DefaultLeaseTTL)
-	d.Set("max_lease_ttl_seconds", mountResp.MaxLeaseTTL)
+	d.Set("default_lease_ttl_seconds", mount.Config.DefaultLeaseTTL)
+	d.Set("max_lease_ttl_seconds", mount.Config.MaxLeaseTTL)
 
 	configPath := fmt.Sprintf("%s/config", d.Id())
 	log.Printf("[DEBUG] Reading %q", configPath)
@@ -605,9 +610,6 @@
 	if raw, ok := d.GetOk("groupfilter"); ok {
 		data["groupfilter"] = raw
 	}
-	if raw, ok := d.GetOk("insecure_tls"); ok {
-		data["insecure_tls"] = raw
-	}
 	if raw, ok := d.GetOk("last_rotation_tolerance"); ok {
 		data["last_rotation_tolerance"] = raw
 	}
@@ -653,6 +655,7 @@
 	if raw, ok := d.GetOk("userdn"); ok {
 		data["userdn"] = raw
 	}
+	data["insecure_tls"] = d.Get("insecure_tls")
 	if _, err := client.Logical().Write(vaultPath, data); err != nil {
 		return fmt.Errorf("error updating template auth backend role %q: %s", vaultPath, err)
 	}

vault/resource_auth_backend.go

@@ -4,6 +4,8 @@
 package vault
 
 import (
+	"context"
+	"errors"
 	"fmt"
 	"log"
 
@@ -13,6 +15,7 @@ import (
 	"github.com/hashicorp/terraform-provider-vault/internal/consts"
 	"github.com/hashicorp/terraform-provider-vault/internal/provider"
 	"github.com/hashicorp/terraform-provider-vault/util"
+	"github.com/hashicorp/terraform-provider-vault/util/mountutil"
 )
 
 func AuthBackendResource() *schema.Resource {
@@ -126,16 +129,17 @@ func authBackendRead(d *schema.ResourceData, meta interface{}) error {
 
 	path := d.Id()
 
-	mount, err := getAuthMountIfPresent(client, path)
-	if err != nil {
-		return err
-	}
-
-	if mount == nil {
+	mount, err := mountutil.GetAuthMount(context.Background(), client, path)
+	if errors.Is(err, mountutil.ErrMountNotFound) {
+		log.Printf("[WARN] Mount %q not found, removing from state.", path)
 		d.SetId("")
 		return nil
 	}
 
+	if err != nil {
+		return err
+	}
+
 	if err := d.Set("type", mount.Type); err != nil {
 		return err
 	}