perf(yaml): Improve large config files loading times (#48)

* TODO

* TODO

* set resolved, update tests

* revert io.readall

* test value from flat key. Consider non string cases

* tests for the code coverage god

* fix(ghw): fix github workflow

---------

Co-authored-by: Adan Urban Reyes <[email protected]>

Author: nhtzr

.github/workflows/build.yaml

@@ -36,3 +36,13 @@ jobs:
         uses: shogo82148/actions-goveralls@v1
         with:
           path-to-profile: profile.cov
+          flag-name: Go-${{ matrix.go }}
+          parallel: true
+
+  finish:
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: shogo82148/actions-goveralls@v1
+        with:
+          parallel-finished: true

pkg/yaml/yaml.go

@@ -2,8 +2,10 @@ package yaml
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"regexp"
+	"strconv"
 	"strings"
 
 	"github.com/armory/go-yaml-tools/pkg/secrets"
@@ -12,141 +14,224 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
+type ObjectMap = map[interface{}]interface{}
+type StringMap = map[string]string
+type OutputMap = map[string]interface{}
+
 // Resolve takes an array of yaml maps and returns a single map of a merged
 // properties.  The order of `ymlTemplates` matters, it should go from lowest
 // to highest precendence.
-func Resolve(ymlTemplates []map[interface{}]interface{}, envKeyPairs map[string]string) (map[string]interface{}, error) {
+func Resolve(ymlTemplates []ObjectMap, envKeyPairs StringMap) (OutputMap, error) {
 	log.Debugf("Using environ %+v\n", envKeyPairs)
 
-	mergedMap := map[interface{}]interface{}{}
+	mergedMap := ObjectMap{}
 	for _, yml := range ymlTemplates {
 		if err := mergo.Merge(&mergedMap, yml, mergo.WithOverride); err != nil {
 			log.Error(err)
 		}
 	}
 
 	// unlike other secret engines, the vault config needs to be registered before it can decrypt anything
-	vaultCfg := extractVaultConfig(mergedMap)
-	if vaultCfg != nil && (secrets.VaultConfig{}) != *vaultCfg {
+	vaultCfg, err := extractVaultConfig(mergedMap)
+	if err == nil {
 		if err := secrets.RegisterVaultConfig(*vaultCfg); err != nil {
 			log.Errorf("Error registering vault config: %v", err)
 		}
 	}
 
 	stringMap := convertToStringMap(mergedMap)
 
-	err := subValues(stringMap, stringMap, envKeyPairs)
-
-	if err != nil {
+	if err := subValues(stringMap, stringMap, envKeyPairs); err != nil {
 		return nil, err
 	}
 
 	return stringMap, nil
 }
 
-func extractVaultConfig(m map[interface{}]interface{}) *secrets.VaultConfig {
-	if secretsMap, ok := m["secrets"].(map[interface{}]interface{}); ok {
-		if vaultmap, ok := secretsMap["vault"].(map[interface{}]interface{}); ok {
-			cfg, err := secrets.DecodeVaultConfig(vaultmap)
-			if err != nil {
-				log.Errorf("Error decoding vault config: %v", err)
-				return nil
-			}
-			return cfg
-		}
+var EVCErrorMissingKey = errors.New("missing secrets.vault key")
+var EVCErrorDecoding = errors.New("error decoding vault config")
+var EVCErrorEmpty = errors.New("empty decoded vault config")
+
+func extractVaultConfig(m ObjectMap) (*secrets.VaultConfig, error) {
+	secretsMap, ok := m["secrets"].(ObjectMap)
+	if !ok {
+		return nil, EVCErrorMissingKey
 	}
-	return nil
+	vaultmap, ok := secretsMap["vault"].(ObjectMap)
+	if !ok {
+		return nil, EVCErrorMissingKey
+	}
+	cfg, err := secrets.DecodeVaultConfig(vaultmap)
+	if err != nil {
+		return nil, fmt.Errorf("%w: %v", EVCErrorDecoding, err)
+	}
+	if cfg == nil {
+		return nil, EVCErrorEmpty
+	}
+	if *cfg == (secrets.VaultConfig{}) {
+		return nil, EVCErrorEmpty
+	}
+	return cfg, nil
 }
 
-func convertToStringMap(m map[interface{}]interface{}) map[string]interface{} {
-	newMap := map[string]interface{}{}
+func convertToStringMap(m ObjectMap) OutputMap {
+	newMap := OutputMap{}
+	var kstring string
 	for k, v := range m {
-		switch v.(type) {
-		case map[interface{}]interface{}:
-			stringMap := convertToStringMap(v.(map[interface{}]interface{}))
-			newMap[k.(string)] = stringMap
-		case []interface{}:
-			var collection []interface{}
-			for _, vv := range v.([]interface{}) {
-				switch vv.(type) {
-				case map[interface{}]interface{}:
-					collection = append(collection, convertToStringMap(vv.(map[interface{}]interface{})))
-				case string, int, bool, float64:
-					collection = append(collection, fmt.Sprintf("%v", vv))
-				}
-			}
-			newMap[k.(string)] = collection
+		kstring = k.(string)
+		convertOneValueToStringMap(v, newMap, kstring)
+	}
+	return newMap
+}
 
-		default:
-			newMap[k.(string)] = fmt.Sprintf("%v", v)
+func convertOneValueToStringMap(v interface{}, newMap OutputMap, kstring string) {
+	switch v := v.(type) {
+	case ObjectMap:
+		newMap[kstring] = convertToStringMap(v)
+	case []interface{}:
+		for i := range v {
+			converOneArrayToStringMap(v[:], i)
 		}
+		newMap[kstring] = v
+	case string:
+		newMap[kstring] = v
+	case int:
+		newMap[kstring] = strconv.Itoa(v)
+	case bool:
+		newMap[kstring] = strconv.FormatBool(v)
+	case float64:
+		newMap[kstring] = strconv.FormatFloat(v, 'g', -1, 64)
+	case float32:
+		newMap[kstring] = strconv.FormatFloat(float64(v), 'g', -1, 64)
+	case fmt.Stringer:
+		newMap[kstring] = v.String()
+	default:
+		newMap[kstring] = fmt.Sprintf("%v", v)
+	}
+}
+
+func converOneArrayToStringMap(v []interface{}, i int) {
+	switch vv := v[i].(type) {
+	case ObjectMap:
+		v[i] = convertToStringMap(vv)
+	case []interface{}:
+		for j := range v {
+			converOneArrayToStringMap(vv[:], j)
+		}
+		v[i] = vv
+	case string:
+		v[i] = vv
+	case int:
+		v[i] = strconv.Itoa(vv)
+	case bool:
+		v[i] = strconv.FormatBool(vv)
+	case float64:
+		v[i] = strconv.FormatFloat(vv, 'g', -1, 64)
+	case float32:
+		v[i] = strconv.FormatFloat(float64(vv), 'g', -1, 64)
+	case fmt.Stringer:
+		v[i] = vv.String()
+	default:
+		v[i] = fmt.Sprintf("%v", vv)
 	}
-	return newMap
 }
 
-func subValues(fullMap map[string]interface{}, subMap map[string]interface{}, env map[string]string) error {
+var re = regexp.MustCompile("\\$\\{(.*?)}")
+
+func subValues(fullMap OutputMap, subMap OutputMap, env StringMap) error {
 	//responsible for finding all variables that need to be substituted
-	keepResolving := true
 	loops := 0
-	re := regexp.MustCompile("\\$\\{(.*?)\\}")
-	for keepResolving && loops < len(subMap) {
+	for loops < len(subMap) {
 		loops++
 		for k, value := range subMap {
-			switch value.(type) {
-			case map[string]interface{}:
-				err := subValues(fullMap, value.(map[string]interface{}), env)
-				if err != nil {
-					return err
-				}
-			case []interface{}:
-				sliceMap := make(map[string]interface{})
-				for i := 0; i < len(value.([]interface{})); i++ {
-					sliceMap[fmt.Sprint(i)] = value.([]interface{})[i]
-				}
-				err := subValues(fullMap, sliceMap, env)
-				if err != nil {
-					return err
-				}
-			case string:
-				valStr := value.(string)
-
-				secret, wasSecret, err := resolveSecret(valStr)
-				if err != nil {
-					return err
-				}
-
-				if wasSecret {
-					subMap[k] = secret
-					continue
-				}
-
-				keys := re.FindAllStringSubmatch(valStr, -1)
-				for _, keyToSub := range keys {
-					resolvedValue := resolveSubs(fullMap, keyToSub[1], env)
-					subMap[k] = strings.Replace(valStr, "${"+keyToSub[1]+"}", resolvedValue, -1)
-				}
+			if err := processOneSubvalue(fullMap, subMap, env, value, k); err != nil {
+				return err
 			}
 		}
 	}
 	return nil
 }
 
-func resolveSecret(valStr string) (string, bool, error) {
-	// if the value is a secret resolve it
-	if secrets.IsEncryptedSecret(valStr) {
-		d, err := secrets.NewDecrypter(context.TODO(), valStr)
+func processOneSubvalue(fullMap OutputMap, subMap OutputMap, env StringMap, value interface{}, k string) error {
+	var secret string
+	var decrypter secrets.Decrypter
+	var valueBytes []byte
+	switch value := value.(type) {
+	case map[string]interface{}:
+		err := subValues(fullMap, value, env)
 		if err != nil {
-			return "", true, err
+			return err
+		}
+	case []interface{}:
+		for i := 0; i < len(value); i++ {
+			err := processOneSubvalueFromArray(fullMap, value[:], env, value[i], i)
+			if err != nil {
+				return err
+			}
 		}
+	case string:
+		if secrets.IsEncryptedSecret(value) {
+			var err error
+			if decrypter, err = secrets.NewDecrypter(context.TODO(), value); err != nil {
+				return err
+			} else if secret, err = decrypter.Decrypt(); err != nil {
+				return err
+			}
+			subMap[k] = secret
+			return nil
+		}
+
+		valueBytes = []byte(value)
+		valueBytes = re.ReplaceAllFunc(valueBytes, func(key []byte) []byte {
+			i := len(key) - 1
+			myKey := string(key[2:i])
+			return []byte(resolveSubs(fullMap, myKey, env))
+		})
+		value = string(valueBytes)
+		subMap[k] = value
+	}
+	return nil
+}
 
-		secret, err := d.Decrypt()
+func processOneSubvalueFromArray(fullMap OutputMap, subslice []interface{}, env StringMap, value interface{}, k int) error {
+	var secret string
+	var decrypter secrets.Decrypter
+	var valueBytes []byte
+	switch value := value.(type) {
+	case map[string]interface{}:
+		err := subValues(fullMap, value, env)
 		if err != nil {
-			return "", true, err
+			return err
+		}
+	case []interface{}:
+		for i := 0; i < len(value); i++ {
+			err := processOneSubvalueFromArray(fullMap, value[:], env, value[i], i)
+			if err != nil {
+				return err
+			}
+		}
+	case string:
+		if secrets.IsEncryptedSecret(value) {
+			var err error
+			if decrypter, err = secrets.NewDecrypter(context.TODO(), value); err != nil {
+				return err
+			} else if secret, err = decrypter.Decrypt(); err != nil {
+				return err
+			}
+			subslice[k] = secret
+			return nil
 		}
 
-		return secret, true, nil
+		valueBytes = []byte(value)
+		valueBytes = re.ReplaceAllFunc(valueBytes, func(key []byte) []byte {
+			i := len(key) - 1
+			myKey := string(key[2:i])
+			return []byte(resolveSubs(fullMap, myKey, env))
+		})
+		value = string(valueBytes)
+		subslice[k] = value
 	}
-	return valStr, false, nil
+	return nil
 }
 
 func resolveSubs(m map[string]interface{}, keyToSub string, env map[string]string) string {
@@ -159,7 +244,7 @@ func resolveSubs(m map[string]interface{}, keyToSub string, env map[string]strin
 		defaultKey = keyDefaultSplit[1]
 	}
 
-	if v := valueFromFlatKey(subKey, m); v != "" {
+	if v, err := valueFromFlatKey(subKey, m); err == nil {
 		defaultKey = v
 	} else if v, ok := env[subKey]; ok {
 		defaultKey = v
@@ -168,17 +253,40 @@ func resolveSubs(m map[string]interface{}, keyToSub string, env map[string]strin
 	return defaultKey
 }
 
-func valueFromFlatKey(flatKey string, m map[string]interface{}) string {
-	keys := strings.Split(flatKey, ".")
-	for _, key := range keys {
-		switch m[key].(type) {
-		case map[string]interface{}:
-			m = m[key].(map[string]interface{})
-		case string:
-			return m[key].(string)
-		default:
-			continue
+var VFFKErrorNotFound = errors.New("not found")
+var VFFKErrorInvalidIntermediaryType = errors.New("expected map[string]interface{}")
+var VFFKErrorInvalidLeafType = errors.New("expected string or stringer()")
+
+func valueFromFlatKey(flatKey string, root map[string]interface{}) (string, error) {
+	fields := strings.Split(flatKey, ".")
+	var currVal interface{} = root
+	var currMap OutputMap // pre-alloc OutputMap ref. Actually assigned & used in loop below
+	var ok bool
+	for i := range fields {
+		if currVal == nil {
+			return "", fmt.Errorf("path %q was %w", flatKey, VFFKErrorNotFound)
+		}
+		if currMap, ok = currVal.(OutputMap); !ok {
+			return "", fmt.Errorf("path %q was of type %T, %w", strings.Join(fields[:i], "."), currVal, VFFKErrorInvalidIntermediaryType)
 		}
+		if currVal, ok = currMap[fields[i]]; !ok {
+			return "", fmt.Errorf("path %q was %w", flatKey, VFFKErrorNotFound)
+		}
+	}
+	switch v := currVal.(type) {
+	case string:
+		return v, nil
+	case fmt.Stringer:
+		return v.String(), nil
+	case float32:
+		return strconv.FormatFloat(float64(v), 'g', -1, 64), nil
+	case float64:
+		return strconv.FormatFloat(v, 'g', -1, 64), nil
+	case int:
+		return strconv.Itoa(v), nil
+	case bool:
+		return strconv.FormatBool(v), nil
+	default:
+		return "", fmt.Errorf("path %q is type %T, %w", flatKey, v, VFFKErrorInvalidLeafType)
 	}
-	return ""
 }