From b42d90d4cef46e581daad7645cd0c54e9099a071 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Agust=C3=ADn=20D=C3=ADaz?= Date: Thu, 5 Oct 2023 12:30:00 -0300 Subject: [PATCH 1/4] feat: add `*WithFilter` helper functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Agustín Díaz --- modules/files/files.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/modules/files/files.go b/modules/files/files.go index c4b58b372..79a7b2df7 100644 --- a/modules/files/files.go +++ b/modules/files/files.go @@ -38,6 +38,15 @@ func IsExistingDir(path string) bool { return err == nil && fileInfo.IsDir() } +func CopyTerraformFolderToDestWithFilter(folderPath string, destRootFolder string, tempFolderPrefix string, fileFilter func(string) bool) (string, error) { + destFolder, err := CopyFolderToDest(folderPath, destRootFolder, tempFolderPrefix, fileFilter) + if err != nil { + return "", err + } + + return destFolder, nil +} + // CopyTerraformFolderToDest creates a copy of the given folder and all its contents in a specified folder with a unique name and the given prefix. // This is useful when running multiple tests in parallel against the same set of Terraform files to ensure the // tests don't overwrite each other's .terraform working directory and terraform.tfstate files. This method returns @@ -56,12 +65,11 @@ func CopyTerraformFolderToDest(folderPath string, destRootFolder string, tempFol return true } - destFolder, err := CopyFolderToDest(folderPath, destRootFolder, tempFolderPrefix, filter) - if err != nil { - return "", err - } + return CopyTerraformFolderToDestWithFilter(folderPath, destRootFolder, tempFolderPrefix, filter) +} - return destFolder, nil +func CopyTerraformFolderToTempWithFilter(folderPath string, tempFolderPrefix string, fileFilter func(string) bool) (string, error) { + return CopyTerraformFolderToDestWithFilter(folderPath, os.TempDir(), tempFolderPrefix, fileFilter) } // CopyTerraformFolderToTemp calls CopyTerraformFolderToDest, passing os.TempDir() as the root destination folder. From c29762ff3e1a495813e3373b14b7154694b873a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Agust=C3=ADn=20D=C3=ADaz?= Date: Thu, 5 Oct 2023 14:02:27 -0300 Subject: [PATCH 2/4] add docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Agustín Díaz --- modules/files/files.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/modules/files/files.go b/modules/files/files.go index 79a7b2df7..a31b54a40 100644 --- a/modules/files/files.go +++ b/modules/files/files.go @@ -38,8 +38,13 @@ func IsExistingDir(path string) bool { return err == nil && fileInfo.IsDir() } -func CopyTerraformFolderToDestWithFilter(folderPath string, destRootFolder string, tempFolderPrefix string, fileFilter func(string) bool) (string, error) { - destFolder, err := CopyFolderToDest(folderPath, destRootFolder, tempFolderPrefix, fileFilter) +// CopyTerraformFolderToDestWithFilter creates a copy of the given folder and all its contents in a specified folder with a unique name and the given prefix. +// This is useful when running multiple tests in parallel against the same set of Terraform files to ensure the +// tests don't overwrite each other's .terraform working directory and terraform.tfstate files. This method returns +// the path to the dest folder with the copied contents. You can configure which files you want to copy by passing in a filter function. +// This method is useful when running through a build tool so the files are copied to a destination that is cleaned on each run of the pipeline. +func CopyTerraformFolderToDestWithFilter(folderPath string, destRootFolder string, tempFolderPrefix string, filter func(string) bool) (string, error) { + destFolder, err := CopyFolderToDest(folderPath, destRootFolder, tempFolderPrefix, filter) if err != nil { return "", err } @@ -68,8 +73,9 @@ func CopyTerraformFolderToDest(folderPath string, destRootFolder string, tempFol return CopyTerraformFolderToDestWithFilter(folderPath, destRootFolder, tempFolderPrefix, filter) } -func CopyTerraformFolderToTempWithFilter(folderPath string, tempFolderPrefix string, fileFilter func(string) bool) (string, error) { - return CopyTerraformFolderToDestWithFilter(folderPath, os.TempDir(), tempFolderPrefix, fileFilter) +// CopyTerraformFolderToTempWithFilter calls CopyTerraformFolderToDestWithFilter, passing os.TempDir() as the root destination folder. +func CopyTerraformFolderToTempWithFilter(folderPath string, tempFolderPrefix string, filter func(string) bool) (string, error) { + return CopyTerraformFolderToDestWithFilter(folderPath, os.TempDir(), tempFolderPrefix, filter) } // CopyTerraformFolderToTemp calls CopyTerraformFolderToDest, passing os.TempDir() as the root destination folder. From 262172d4d3db0ee436b4643d313124985bf63485 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Agust=C3=ADn=20D=C3=ADaz?= Date: Thu, 5 Oct 2023 14:14:45 -0300 Subject: [PATCH 3/4] feat: add *WithFilter helper functions to test_structure module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Agustín Díaz --- modules/files/files.go | 22 ++++---- modules/test-structure/test_structure.go | 72 +++++++++++++++++++++++- 2 files changed, 80 insertions(+), 14 deletions(-) diff --git a/modules/files/files.go b/modules/files/files.go index a31b54a40..bf56ade8c 100644 --- a/modules/files/files.go +++ b/modules/files/files.go @@ -10,6 +10,16 @@ import ( "github.com/mattn/go-zglob" ) +func DefaultPathFilter(path string) bool { + if PathIsTerraformVersionFile(path) || PathIsTerraformLockFile(path) { + return true + } + if PathContainsHiddenFileOrFolder(path) || PathContainsTerraformStateOrVars(path) { + return false + } + return true +} + // FileExists returns true if the given file exists. func FileExists(path string) bool { _, err := os.Stat(path) @@ -60,17 +70,7 @@ func CopyTerraformFolderToDestWithFilter(folderPath string, destRootFolder strin // files, and terraform.tfvars files are not copied to this temp folder, as you typically don't want them interfering with your tests. // This method is useful when running through a build tool so the files are copied to a destination that is cleaned on each run of the pipeline. func CopyTerraformFolderToDest(folderPath string, destRootFolder string, tempFolderPrefix string) (string, error) { - filter := func(path string) bool { - if PathIsTerraformVersionFile(path) || PathIsTerraformLockFile(path) { - return true - } - if PathContainsHiddenFileOrFolder(path) || PathContainsTerraformStateOrVars(path) { - return false - } - return true - } - - return CopyTerraformFolderToDestWithFilter(folderPath, destRootFolder, tempFolderPrefix, filter) + return CopyTerraformFolderToDestWithFilter(folderPath, destRootFolder, tempFolderPrefix, DefaultPathFilter) } // CopyTerraformFolderToTempWithFilter calls CopyTerraformFolderToDestWithFilter, passing os.TempDir() as the root destination folder. diff --git a/modules/test-structure/test_structure.go b/modules/test-structure/test_structure.go index 032716e6e..5640887b7 100644 --- a/modules/test-structure/test_structure.go +++ b/modules/test-structure/test_structure.go @@ -43,6 +43,37 @@ func SkipStageEnvVarSet() bool { return false } +// CopyTerraformFolderToTempWithFilter copies the given root folder to a randomly-named temp folder and return the path to the +// given terraform modules folder within the new temp root folder. This is useful when running multiple tests in +// parallel against the same set of Terraform files to ensure the tests don't overwrite each other's .terraform working +// directory and terraform.tfstate files. To ensure relative paths work, we copy over the entire root folder to a temp +// folder, and then return the path within that temp folder to the given terraform module dir, which is where the actual +// test will be running. +// For example, suppose you had the target terraform folder you want to test in "/examples/terraform-aws-example" +// relative to the repo root. If your tests reside in the "/test" relative to the root, then you will use this as +// follows: +// +// // Root folder where terraform files should be (relative to the test folder) +// rootFolder := ".." +// +// // Relative path to terraform module being tested from the root folder +// terraformFolderRelativeToRoot := "examples/terraform-aws-example" +// +// // Copy the terraform folder to a temp folder +// tempTestFolder := test_structure.CopyTerraformFolderToTemp(t, rootFolder, terraformFolderRelativeToRoot) +// +// // Make sure to use the temp test folder in the terraform options +// terraformOptions := &terraform.Options{ +// TerraformDir: tempTestFolder, +// } +// +// Note that if any of the SKIP_ environment variables is set, we assume this is a test in the local dev where +// there are no other concurrent tests running and we want to be able to cache test data between test stages, so in that +// case, we do NOT copy anything to a temp folder, and return the path to the original terraform module folder instead. +func CopyTerraformFolderToTempWithFilter(t testing.TestingT, rootFolder string, terraformModuleFolder string, filter func(string) bool) string { + return CopyTerraformFolderToDestWithFilter(t, rootFolder, terraformModuleFolder, os.TempDir(), filter) +} + // CopyTerraformFolderToTemp copies the given root folder to a randomly-named temp folder and return the path to the // given terraform modules folder within the new temp root folder. This is useful when running multiple tests in // parallel against the same set of Terraform files to ensure the tests don't overwrite each other's .terraform working @@ -74,7 +105,7 @@ func CopyTerraformFolderToTemp(t testing.TestingT, rootFolder string, terraformM return CopyTerraformFolderToDest(t, rootFolder, terraformModuleFolder, os.TempDir()) } -// CopyTerraformFolderToDest copies the given root folder to a randomly-named temp folder and return the path to the +// CopyTerraformFolderToDestWithFilter copies the given root folder to a randomly-named temp folder and return the path to the // given terraform modules folder within the new temp root folder. This is useful when running multiple tests in // parallel against the same set of Terraform files to ensure the tests don't overwrite each other's .terraform working // directory and terraform.tfstate files. To ensure relative paths work, we copy over the entire root folder to a temp @@ -105,7 +136,7 @@ func CopyTerraformFolderToTemp(t testing.TestingT, rootFolder string, terraformM // Note that if any of the SKIP_ environment variables is set, we assume this is a test in the local dev where // there are no other concurrent tests running and we want to be able to cache test data between test stages, so in that // case, we do NOT copy anything to a temp folder, and return the path to the original terraform module folder instead. -func CopyTerraformFolderToDest(t testing.TestingT, rootFolder string, terraformModuleFolder string, destRootFolder string) string { +func CopyTerraformFolderToDestWithFilter(t testing.TestingT, rootFolder string, terraformModuleFolder string, destRootFolder string, filter func(string) bool) string { if SkipStageEnvVarSet() { logger.Logf(t, "A SKIP_XXX environment variable is set. Using original examples folder rather than a temp folder so we can cache data between stages for faster local testing.") return filepath.Join(rootFolder, terraformModuleFolder) @@ -119,7 +150,7 @@ func CopyTerraformFolderToDest(t testing.TestingT, rootFolder string, terraformM t.Fatal(files.DirNotFoundError{Directory: fullTerraformModuleFolder}) } - tmpRootFolder, err := files.CopyTerraformFolderToDest(rootFolder, destRootFolder, cleanName(t.Name())) + tmpRootFolder, err := files.CopyTerraformFolderToDestWithFilter(rootFolder, destRootFolder, cleanName(t.Name()), filter) if err != nil { t.Fatal(err) } @@ -132,6 +163,41 @@ func CopyTerraformFolderToDest(t testing.TestingT, rootFolder string, terraformM return tmpTestFolder } +// CopyTerraformFolderToDest copies the given root folder to a randomly-named temp folder and return the path to the +// given terraform modules folder within the new temp root folder. This is useful when running multiple tests in +// parallel against the same set of Terraform files to ensure the tests don't overwrite each other's .terraform working +// directory and terraform.tfstate files. To ensure relative paths work, we copy over the entire root folder to a temp +// folder, and then return the path within that temp folder to the given terraform module dir, which is where the actual +// test will be running. +// For example, suppose you had the target terraform folder you want to test in "/examples/terraform-aws-example" +// relative to the repo root. If your tests reside in the "/test" relative to the root, then you will use this as +// follows: +// +// // Destination for the copy of the files. In this example we are using the Azure Dev Ops variable +// // for the folder that is cleaned after each pipeline job. +// destRootFolder := os.Getenv("AGENT_TEMPDIRECTORY") +// +// // Root folder where terraform files should be (relative to the test folder) +// rootFolder := ".." +// +// // Relative path to terraform module being tested from the root folder +// terraformFolderRelativeToRoot := "examples/terraform-aws-example" +// +// // Copy the terraform folder to a temp folder +// tempTestFolder := test_structure.CopyTerraformFolderToTemp(t, rootFolder, terraformFolderRelativeToRoot, destRootFolder) +// +// // Make sure to use the temp test folder in the terraform options +// terraformOptions := &terraform.Options{ +// TerraformDir: tempTestFolder, +// } +// +// Note that if any of the SKIP_ environment variables is set, we assume this is a test in the local dev where +// there are no other concurrent tests running and we want to be able to cache test data between test stages, so in that +// case, we do NOT copy anything to a temp folder, and return the path to the original terraform module folder instead. +func CopyTerraformFolderToDest(t testing.TestingT, rootFolder string, terraformModuleFolder string, destRootFolder string) string { + return CopyTerraformFolderToDestWithFilter(t, rootFolder, terraformModuleFolder, destRootFolder, files.DefaultPathFilter) +} + func cleanName(originalName string) string { parts := strings.Split(originalName, "/") return parts[len(parts)-1] From 3504f865b0a89fb34549e65fe9ffbf2031de446b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Agust=C3=ADn=20D=C3=ADaz?= Date: Thu, 5 Oct 2023 14:16:38 -0300 Subject: [PATCH 4/4] docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Agustín Díaz --- modules/files/files.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/files/files.go b/modules/files/files.go index bf56ade8c..e7e9921d9 100644 --- a/modules/files/files.go +++ b/modules/files/files.go @@ -10,6 +10,7 @@ import ( "github.com/mattn/go-zglob" ) +// DefaultPathFilter is the default filter used by CopyTerraformFolderToDest to determine which files to copy. func DefaultPathFilter(path string) bool { if PathIsTerraformVersionFile(path) || PathIsTerraformLockFile(path) { return true