From 7d289d518008e02c7e64a887485638f09b05c267 Mon Sep 17 00:00:00 2001 From: Maidul Islam Date: Fri, 25 Nov 2022 17:51:28 -0500 Subject: [PATCH 1/2] rough imp, unable to debug further recursion --- cli/packages/models/error.go | 14 ++++ cli/packages/util/secrets.go | 73 ++++++++++++++++++ cli/packages/util/secrets_test.go | 122 ++++++++++++++++++++++++++++++ 3 files changed, 209 insertions(+) create mode 100644 cli/packages/models/error.go create mode 100644 cli/packages/util/secrets_test.go diff --git a/cli/packages/models/error.go b/cli/packages/models/error.go new file mode 100644 index 0000000000..28e48d54d2 --- /dev/null +++ b/cli/packages/models/error.go @@ -0,0 +1,14 @@ +package models + +import log "github.com/sirupsen/logrus" + +// Custom error type so that we can give helpful messages in CLI +type Error struct { + Err error + DebugMessage string + FriendlyMessage string +} + +func (e *Error) printFriendlyMessage() { + log.Infoln(e.FriendlyMessage) +} diff --git a/cli/packages/util/secrets.go b/cli/packages/util/secrets.go index 475640713e..d38ec58054 100644 --- a/cli/packages/util/secrets.go +++ b/cli/packages/util/secrets.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "errors" "fmt" + "regexp" "strings" "github.com/Infisical/infisical-merge/packages/models" @@ -205,3 +206,75 @@ func GetWorkSpacesFromAPI(userCreds models.UserCredentials) (workspaces []models return getWorkSpacesResponse.Workspaces, nil } + +func getExpandedEnvVariable(secrets []models.SingleEnvironmentVariable, variableWeAreLookingFor string, hashMapOfCompleteVariables map[string]string, hashMapOfSelfRefs map[string]string) string { + if value, found := hashMapOfCompleteVariables[variableWeAreLookingFor]; found { + return value + } + + for _, secret := range secrets { + if secret.Key == variableWeAreLookingFor { + regex := regexp.MustCompile(`\${([^\}]*)}`) + variablesToPopulate := regex.FindAllString(secret.Value, -1) + + // case: variable is a constant so return its value + if len(variablesToPopulate) == 0 { + return secret.Value + } + + fullyReplacedValue := secret.Value + fmt.Println("variablesToPopulate", variablesToPopulate) + for _, variableWithSign := range variablesToPopulate { + variableWithoutSign := strings.Trim(variableWithSign, "}") + variableWithoutSign = strings.Trim(variableWithoutSign, "${") + + // case: reference to self + if variableWithoutSign == secret.Key { + hashMapOfSelfRefs[variableWithoutSign] = variableWithoutSign + continue + } else { + var expandedVariableValue string + + if preComputedVariable, found := hashMapOfCompleteVariables[variableWithoutSign]; found { + fmt.Println("precompute for varable: ", variableWithoutSign) + expandedVariableValue = preComputedVariable + } else { + fmt.Println("compute for varable: ", variableWithoutSign) + expandedVariableValue = getExpandedEnvVariable(secrets, variableWithoutSign, hashMapOfCompleteVariables, hashMapOfSelfRefs) + hashMapOfCompleteVariables[variableWithoutSign] = expandedVariableValue + } + + // If after expanding all the vars above, is the current var a self ref? if so no replacement needed for it + if _, found := hashMapOfSelfRefs[variableWithoutSign]; found { + continue + } else { + fullyReplacedValue = strings.ReplaceAll(fullyReplacedValue, variableWithSign, expandedVariableValue) + } + } + + return fullyReplacedValue + } + } else { + continue + } + } + + return "${" + variableWeAreLookingFor + "}" +} + +func SubstituteSecrets(secrets []models.SingleEnvironmentVariable) []models.SingleEnvironmentVariable { + hashMapOfCompleteVariables := make(map[string]string) + hashMapOfSelfRefs := make(map[string]string) + expandedSecrets := []models.SingleEnvironmentVariable{} + for _, secret := range secrets { + expandedVariable := getExpandedEnvVariable(secrets, secret.Key, hashMapOfCompleteVariables, hashMapOfSelfRefs) + fmt.Println(secret.Key, "=", expandedVariable) + expandedSecrets = append(expandedSecrets, models.SingleEnvironmentVariable{ + Key: secret.Key, + Value: expandedVariable, + }) + + } + + return expandedSecrets +} diff --git a/cli/packages/util/secrets_test.go b/cli/packages/util/secrets_test.go new file mode 100644 index 0000000000..02e0c0a836 --- /dev/null +++ b/cli/packages/util/secrets_test.go @@ -0,0 +1,122 @@ +package util + +import ( + "testing" + + "github.com/Infisical/infisical-merge/packages/models" +) + +// References to self should return the value unaltered +// func Test_SubstituteSecrets_When_ReferenceToSelf(t *testing.T) { + +// var tests = []struct { +// Key string +// Value string +// ExpectedValue string +// }{ +// {Key: "A", Value: "${A}", ExpectedValue: "${A}"}, +// {Key: "A", Value: "${A} ${A}", ExpectedValue: "${A} ${A}"}, +// {Key: "A", Value: "${A}${A}", ExpectedValue: "${A}${A}"}, +// } + +// for _, test := range tests { +// secret := models.SingleEnvironmentVariable{ +// Key: test.Key, +// Value: test.Value, +// } + +// secrets := []models.SingleEnvironmentVariable{secret} +// result := SubstituteSecrets(secrets) + +// if result[0].Value != test.ExpectedValue { +// t.Errorf("Test_SubstituteSecrets_When_ReferenceToSelf: expected %s but got %s for input %s", test.ExpectedValue, result[0].Value, test.Value) +// } + +// } +// } + +// func Test_SubstituteSecrets_When_ReferenceDoesNotExist(t *testing.T) { + +// var tests = []struct { +// Key string +// Value string +// ExpectedValue string +// }{ +// {Key: "A", Value: "${X}", ExpectedValue: "${X}"}, +// {Key: "A", Value: "${H}HELLO", ExpectedValue: "${H}HELLO"}, +// {Key: "A", Value: "${L}${S}", ExpectedValue: "${L}${S}"}, +// } + +// for _, test := range tests { +// secret := models.SingleEnvironmentVariable{ +// Key: test.Key, +// Value: test.Value, +// } + +// secrets := []models.SingleEnvironmentVariable{secret} +// result := SubstituteSecrets(secrets) + +// if result[0].Value != test.ExpectedValue { +// t.Errorf("Test_SubstituteSecrets_When_ReferenceToSelf: expected %s but got %s for input %s", test.ExpectedValue, result[0].Value, test.Value) +// } + +// } +// } + +func Test_SubstituteSecrets_When_ReferenceDoesNotExist_And_Self_Referencing(t *testing.T) { + + tests := []struct { + Key string + Value string + ExpectedValue string + }{ + { + Key: "A", + Value: "*${A}* ${X}", + ExpectedValue: "*${A}*", + }, + { + Key: "H", + Value: "${X} >>>", + ExpectedValue: "*${A}*", + }, + { + Key: "X", + Value: "DOMAIN", + ExpectedValue: "DOMAIN", + }, + { + Key: "P", + Value: "${X} === ${A} ${H}", + ExpectedValue: "DOMAIN", + }, + // { + // Key: "B", + // Value: "*${A}*TOKEN*${X}*", + // ExpectedValue: "*${A}*TOKEN*DOMAIN*", + // }, + // { + // Key: "C", + // Value: "*${A}* *${X}* *${B}* *${UNKNOWN}*", + // ExpectedValue: "*${A}* *DOMAIN* **${A}*TOKEN*DOMAIN** *${UNKNOWN}*", + // }, + // { + // Key: "W", + // Value: "*${W}* ${LOL $JK} *${C}* *${C}*", + // ExpectedValue: "*${W}* ${LOL $JK} **${A}* *DOMAIN* **${A}*TOKEN*DOMAIN** *${UNKNOWN}** **${A}* *DOMAIN* **${A}*TOKEN*DOMAIN** *${UNKNOWN}**", + // }, + } + + secrets := []models.SingleEnvironmentVariable{} + for _, test := range tests { + secrets = append(secrets, models.SingleEnvironmentVariable{Key: test.Key, Value: test.Value}) + } + + SubstituteSecrets(secrets) + + // if result[0].Value != test.ExpectedValue { + // t.Errorf("Test_SubstituteSecrets_When_ReferenceToSelf: expected %s but got %s for input %s", test.ExpectedValue, result[0].Value, test.Value) + // } + + // fmt.Println(result) +} From 746ded9a539b141273cfd678afeb8975631bbdce Mon Sep 17 00:00:00 2001 From: Maidul Islam Date: Sat, 26 Nov 2022 16:54:36 -0500 Subject: [PATCH 2/2] Add substitute flag for run --- cli/packages/cmd/run.go | 16 ++- cli/packages/util/secrets.go | 14 +- cli/packages/util/secrets_test.go | 204 ++++++++++++++++++------------ 3 files changed, 142 insertions(+), 92 deletions(-) diff --git a/cli/packages/cmd/run.go b/cli/packages/cmd/run.go index 1c07f06c8c..d9ed47a365 100644 --- a/cli/packages/cmd/run.go +++ b/cli/packages/cmd/run.go @@ -33,6 +33,13 @@ var runCmd = &cobra.Command{ return } + substitute, err := cmd.Flags().GetBool("substitute") + if err != nil { + log.Errorln("Unable to parse the substitute flag") + log.Debugln(err) + return + } + projectId, err := cmd.Flags().GetString("projectId") if err != nil { log.Errorln("Unable to parse the project id flag") @@ -82,7 +89,13 @@ var runCmd = &cobra.Command{ } } - execCmd(args[0], args[1:], envsFromApi) + if substitute { + substitutions := util.SubstituteSecrets(envsFromApi) + execCmd(args[0], args[1:], substitutions) + } else { + execCmd(args[0], args[1:], envsFromApi) + } + }, } @@ -90,6 +103,7 @@ func init() { rootCmd.AddCommand(runCmd) runCmd.Flags().StringP("env", "e", "dev", "Set the environment (dev, prod, etc.) from which your secrets should be pulled from") runCmd.Flags().String("projectId", "", "The project ID from which your secrets should be pulled from") + runCmd.Flags().Bool("substitute", true, "Parse shell variable substitutions in your secrets") } // Credit: inspired by AWS Valut diff --git a/cli/packages/util/secrets.go b/cli/packages/util/secrets.go index d38ec58054..5cf76d48ef 100644 --- a/cli/packages/util/secrets.go +++ b/cli/packages/util/secrets.go @@ -222,8 +222,7 @@ func getExpandedEnvVariable(secrets []models.SingleEnvironmentVariable, variable return secret.Value } - fullyReplacedValue := secret.Value - fmt.Println("variablesToPopulate", variablesToPopulate) + valueToEdit := secret.Value for _, variableWithSign := range variablesToPopulate { variableWithoutSign := strings.Trim(variableWithSign, "}") variableWithoutSign = strings.Trim(variableWithoutSign, "${") @@ -236,10 +235,8 @@ func getExpandedEnvVariable(secrets []models.SingleEnvironmentVariable, variable var expandedVariableValue string if preComputedVariable, found := hashMapOfCompleteVariables[variableWithoutSign]; found { - fmt.Println("precompute for varable: ", variableWithoutSign) expandedVariableValue = preComputedVariable } else { - fmt.Println("compute for varable: ", variableWithoutSign) expandedVariableValue = getExpandedEnvVariable(secrets, variableWithoutSign, hashMapOfCompleteVariables, hashMapOfSelfRefs) hashMapOfCompleteVariables[variableWithoutSign] = expandedVariableValue } @@ -248,12 +245,13 @@ func getExpandedEnvVariable(secrets []models.SingleEnvironmentVariable, variable if _, found := hashMapOfSelfRefs[variableWithoutSign]; found { continue } else { - fullyReplacedValue = strings.ReplaceAll(fullyReplacedValue, variableWithSign, expandedVariableValue) + valueToEdit = strings.ReplaceAll(valueToEdit, variableWithSign, expandedVariableValue) } } - - return fullyReplacedValue } + + return valueToEdit + } else { continue } @@ -266,9 +264,9 @@ func SubstituteSecrets(secrets []models.SingleEnvironmentVariable) []models.Sing hashMapOfCompleteVariables := make(map[string]string) hashMapOfSelfRefs := make(map[string]string) expandedSecrets := []models.SingleEnvironmentVariable{} + for _, secret := range secrets { expandedVariable := getExpandedEnvVariable(secrets, secret.Key, hashMapOfCompleteVariables, hashMapOfSelfRefs) - fmt.Println(secret.Key, "=", expandedVariable) expandedSecrets = append(expandedSecrets, models.SingleEnvironmentVariable{ Key: secret.Key, Value: expandedVariable, diff --git a/cli/packages/util/secrets_test.go b/cli/packages/util/secrets_test.go index 02e0c0a836..513e4f7e3b 100644 --- a/cli/packages/util/secrets_test.go +++ b/cli/packages/util/secrets_test.go @@ -7,61 +7,61 @@ import ( ) // References to self should return the value unaltered -// func Test_SubstituteSecrets_When_ReferenceToSelf(t *testing.T) { - -// var tests = []struct { -// Key string -// Value string -// ExpectedValue string -// }{ -// {Key: "A", Value: "${A}", ExpectedValue: "${A}"}, -// {Key: "A", Value: "${A} ${A}", ExpectedValue: "${A} ${A}"}, -// {Key: "A", Value: "${A}${A}", ExpectedValue: "${A}${A}"}, -// } - -// for _, test := range tests { -// secret := models.SingleEnvironmentVariable{ -// Key: test.Key, -// Value: test.Value, -// } - -// secrets := []models.SingleEnvironmentVariable{secret} -// result := SubstituteSecrets(secrets) - -// if result[0].Value != test.ExpectedValue { -// t.Errorf("Test_SubstituteSecrets_When_ReferenceToSelf: expected %s but got %s for input %s", test.ExpectedValue, result[0].Value, test.Value) -// } - -// } -// } - -// func Test_SubstituteSecrets_When_ReferenceDoesNotExist(t *testing.T) { - -// var tests = []struct { -// Key string -// Value string -// ExpectedValue string -// }{ -// {Key: "A", Value: "${X}", ExpectedValue: "${X}"}, -// {Key: "A", Value: "${H}HELLO", ExpectedValue: "${H}HELLO"}, -// {Key: "A", Value: "${L}${S}", ExpectedValue: "${L}${S}"}, -// } - -// for _, test := range tests { -// secret := models.SingleEnvironmentVariable{ -// Key: test.Key, -// Value: test.Value, -// } - -// secrets := []models.SingleEnvironmentVariable{secret} -// result := SubstituteSecrets(secrets) - -// if result[0].Value != test.ExpectedValue { -// t.Errorf("Test_SubstituteSecrets_When_ReferenceToSelf: expected %s but got %s for input %s", test.ExpectedValue, result[0].Value, test.Value) -// } - -// } -// } +func Test_SubstituteSecrets_When_ReferenceToSelf(t *testing.T) { + + var tests = []struct { + Key string + Value string + ExpectedValue string + }{ + {Key: "A", Value: "${A}", ExpectedValue: "${A}"}, + {Key: "A", Value: "${A} ${A}", ExpectedValue: "${A} ${A}"}, + {Key: "A", Value: "${A}${A}", ExpectedValue: "${A}${A}"}, + } + + for _, test := range tests { + secret := models.SingleEnvironmentVariable{ + Key: test.Key, + Value: test.Value, + } + + secrets := []models.SingleEnvironmentVariable{secret} + result := SubstituteSecrets(secrets) + + if result[0].Value != test.ExpectedValue { + t.Errorf("Test_SubstituteSecrets_When_ReferenceToSelf: expected %s but got %s for input %s", test.ExpectedValue, result[0].Value, test.Value) + } + + } +} + +func Test_SubstituteSecrets_When_ReferenceDoesNotExist(t *testing.T) { + + var tests = []struct { + Key string + Value string + ExpectedValue string + }{ + {Key: "A", Value: "${X}", ExpectedValue: "${X}"}, + {Key: "A", Value: "${H}HELLO", ExpectedValue: "${H}HELLO"}, + {Key: "A", Value: "${L}${S}", ExpectedValue: "${L}${S}"}, + } + + for _, test := range tests { + secret := models.SingleEnvironmentVariable{ + Key: test.Key, + Value: test.Value, + } + + secrets := []models.SingleEnvironmentVariable{secret} + result := SubstituteSecrets(secrets) + + if result[0].Value != test.ExpectedValue { + t.Errorf("Test_SubstituteSecrets_When_ReferenceToSelf: expected %s but got %s for input %s", test.ExpectedValue, result[0].Value, test.Value) + } + + } +} func Test_SubstituteSecrets_When_ReferenceDoesNotExist_And_Self_Referencing(t *testing.T) { @@ -70,41 +70,41 @@ func Test_SubstituteSecrets_When_ReferenceDoesNotExist_And_Self_Referencing(t *t Value string ExpectedValue string }{ + { + Key: "O", + Value: "${P} ==$$ ${X} ${UNKNOWN} ${A}", + ExpectedValue: "DOMAIN === ${A} DOMAIN >>> ==$$ DOMAIN ${UNKNOWN} ${A}", + }, + { + Key: "X", + Value: "DOMAIN", + ExpectedValue: "DOMAIN", + }, { Key: "A", Value: "*${A}* ${X}", - ExpectedValue: "*${A}*", + ExpectedValue: "*${A}* DOMAIN", }, { Key: "H", Value: "${X} >>>", - ExpectedValue: "*${A}*", + ExpectedValue: "DOMAIN >>>", }, { - Key: "X", - Value: "DOMAIN", - ExpectedValue: "DOMAIN", + Key: "P", + Value: "DOMAIN === ${A} ${H}", + ExpectedValue: "DOMAIN === ${A} DOMAIN >>>", }, { - Key: "P", - Value: "${X} === ${A} ${H}", - ExpectedValue: "DOMAIN", + Key: "T", + Value: "${P} ==$$ ${X} ${UNKNOWN} ${A} ${P} ==$$ ${X} ${UNKNOWN} ${A}", + ExpectedValue: "DOMAIN === ${A} DOMAIN >>> ==$$ DOMAIN ${UNKNOWN} ${A} DOMAIN === ${A} DOMAIN >>> ==$$ DOMAIN ${UNKNOWN} ${A}", + }, + { + Key: "S", + Value: "${ SSS$$ ${HEY}", + ExpectedValue: "${ SSS$$ ${HEY}", }, - // { - // Key: "B", - // Value: "*${A}*TOKEN*${X}*", - // ExpectedValue: "*${A}*TOKEN*DOMAIN*", - // }, - // { - // Key: "C", - // Value: "*${A}* *${X}* *${B}* *${UNKNOWN}*", - // ExpectedValue: "*${A}* *DOMAIN* **${A}*TOKEN*DOMAIN** *${UNKNOWN}*", - // }, - // { - // Key: "W", - // Value: "*${W}* ${LOL $JK} *${C}* *${C}*", - // ExpectedValue: "*${W}* ${LOL $JK} **${A}* *DOMAIN* **${A}*TOKEN*DOMAIN** *${UNKNOWN}** **${A}* *DOMAIN* **${A}*TOKEN*DOMAIN** *${UNKNOWN}**", - // }, } secrets := []models.SingleEnvironmentVariable{} @@ -112,11 +112,49 @@ func Test_SubstituteSecrets_When_ReferenceDoesNotExist_And_Self_Referencing(t *t secrets = append(secrets, models.SingleEnvironmentVariable{Key: test.Key, Value: test.Value}) } - SubstituteSecrets(secrets) + results := SubstituteSecrets(secrets) + + for index, expanded := range results { + if expanded.Value != tests[index].ExpectedValue { + t.Errorf("Test_SubstituteSecrets_When_ReferenceToSelf: expected [%s] but got [%s] for input [%s]", tests[index].ExpectedValue, expanded.Value, tests[index].Value) + } + } +} + +func Test_SubstituteSecrets_When_No_SubstituteNeeded(t *testing.T) { + + tests := []struct { + Key string + Value string + ExpectedValue string + }{ + { + Key: "DOMAIN", + Value: "infisical.com", + ExpectedValue: "infisical.com", + }, + { + Key: "API_KEY", + Value: "hdgsvjshcgkdckhevdkd", + ExpectedValue: "hdgsvjshcgkdckhevdkd", + }, + { + Key: "ENV", + Value: "PROD", + ExpectedValue: "PROD", + }, + } + + secrets := []models.SingleEnvironmentVariable{} + for _, test := range tests { + secrets = append(secrets, models.SingleEnvironmentVariable{Key: test.Key, Value: test.Value}) + } - // if result[0].Value != test.ExpectedValue { - // t.Errorf("Test_SubstituteSecrets_When_ReferenceToSelf: expected %s but got %s for input %s", test.ExpectedValue, result[0].Value, test.Value) - // } + results := SubstituteSecrets(secrets) - // fmt.Println(result) + for index, expanded := range results { + if expanded.Value != tests[index].ExpectedValue { + t.Errorf("Test_SubstituteSecrets_When_ReferenceToSelf: expected [%s] but got [%s] for input [%s]", tests[index].ExpectedValue, expanded.Value, tests[index].Value) + } + } }