diff --git a/acceptance/framework/helpers/helpers.go b/acceptance/framework/helpers/helpers.go index a9636be30e..a2d98264c8 100644 --- a/acceptance/framework/helpers/helpers.go +++ b/acceptance/framework/helpers/helpers.go @@ -8,6 +8,7 @@ import ( "encoding/json" "fmt" "os" + "os/exec" "os/signal" "strings" "syscall" @@ -15,9 +16,12 @@ import ( "time" "github.com/gruntwork-io/terratest/modules/helm" + "github.com/gruntwork-io/terratest/modules/k8s" + terratestLogger "github.com/gruntwork-io/terratest/modules/logger" "github.com/gruntwork-io/terratest/modules/random" "github.com/hashicorp/consul-k8s/acceptance/framework/logger" "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/sdk/testutil" "github.com/hashicorp/consul/sdk/testutil/retry" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -185,3 +189,135 @@ func RegisterExternalService(t *testing.T, consulClient *api.Client, namespace, }, nil) require.NoError(t, err) } + +type Command struct { + Command string // The command to run + Args []string // The args to pass to the command + WorkingDir string // The working directory + Env map[string]string // Additional environment variables to set + Logger *terratestLogger.Logger +} + +type cmdResult struct { + output string + err error +} + +func RunCommand(t testutil.TestingTB, options *k8s.KubectlOptions, command Command) (string, error) { + t.Helper() + + resultCh := make(chan *cmdResult, 1) + + go func() { + output, err := exec.Command(command.Command, command.Args...).CombinedOutput() + resultCh <- &cmdResult{output: string(output), err: err} + }() + + // might not be needed + for _, arg := range command.Args { + if strings.Contains(arg, "delete") { + go func() { + GetCRDRemoveFinalizers(t, options) + }() + } + } + + select { + case res := <-resultCh: + if res.err != nil { + logger.Logf(t, "Output: %v.", res.output) + } + return res.output, res.err + // Sometimes this func runs for too long handle timeout if needed. + case <-time.After(320 * time.Second): + GetCRDRemoveFinalizers(t, options) + logger.Logf(t, "RunCommand timed out") + return "", nil + } +} + +// getCRDRemoveFinalizers gets CRDs with finalizers and removes them. +func GetCRDRemoveFinalizers(t testutil.TestingTB, options *k8s.KubectlOptions) { + t.Helper() + crdNames, err := getCRDsWithFinalizers(options) + if err != nil { + logger.Logf(t, "Unable to get CRDs with finalizers, %v.", err) + } + + if len(crdNames) > 0 { + removeFinalizers(t, options, crdNames) + } +} + +// CRD struct to parse CRD JSON output. +type CRD struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + Finalizers []string `json:"finalizers"` + } `json:"metadata"` + } `json:"items"` +} + +func getCRDsWithFinalizers(options *k8s.KubectlOptions) ([]string, error) { + cmdArgs := createCmdArgs(options) + args := []string{"get", "crd", "-o=json"} + + cmdArgs = append(cmdArgs, args...) + command := Command{ + Command: "kubectl", + Args: cmdArgs, + Env: options.Env, + } + + output, err := exec.Command(command.Command, command.Args...).CombinedOutput() + + var crds CRD + if err := json.Unmarshal(output, &crds); err != nil { + return nil, fmt.Errorf("error parsing JSON: %v", err) + } + + var crdNames []string + for _, item := range crds.Items { + if len(item.Metadata.Finalizers) > 0 { + crdNames = append(crdNames, item.Metadata.Name) + } + } + + return crdNames, err +} + +// removeFinalizers removes finalizers from CRDs. +func removeFinalizers(t testutil.TestingTB, options *k8s.KubectlOptions, crdNames []string) { + cmdArgs := createCmdArgs(options) + for _, crd := range crdNames { + args := []string{"patch", "crd", crd, "--type=json", "-p=[{\"op\": \"remove\", \"path\": \"/metadata/finalizers\"}]"} + + cmdArgs = append(cmdArgs, args...) + command := Command{ + Command: "kubectl", + Args: cmdArgs, + Env: options.Env, + } + + _, err := exec.Command(command.Command, command.Args...).CombinedOutput() + if err != nil { + logger.Logf(t, "Unable to remove finalizers, proceeding anyway: %v.", err) + } + fmt.Printf("Finalizers removed from CRD %s\n", crd) + } +} + +func createCmdArgs(options *k8s.KubectlOptions) []string { + var cmdArgs []string + if options.ContextName != "" { + cmdArgs = append(cmdArgs, "--context", options.ContextName) + } + if options.ConfigPath != "" { + cmdArgs = append(cmdArgs, "--kubeconfig", options.ConfigPath) + } + if options.Namespace != "" { + cmdArgs = append(cmdArgs, "--namespace", options.Namespace) + } + return cmdArgs +} diff --git a/acceptance/framework/k8s/kubectl.go b/acceptance/framework/k8s/kubectl.go index 4416c7e6a9..586389ead5 100644 --- a/acceptance/framework/k8s/kubectl.go +++ b/acceptance/framework/k8s/kubectl.go @@ -10,7 +10,7 @@ import ( "github.com/gruntwork-io/terratest/modules/k8s" terratestLogger "github.com/gruntwork-io/terratest/modules/logger" - "github.com/gruntwork-io/terratest/modules/shell" + "github.com/hashicorp/consul-k8s/acceptance/framework/helpers" "github.com/hashicorp/consul-k8s/acceptance/framework/logger" "github.com/hashicorp/consul/sdk/testutil/retry" "github.com/stretchr/testify/require" @@ -48,7 +48,7 @@ func RunKubectlAndGetOutputWithLoggerE(t *testing.T, options *k8s.KubectlOptions cmdArgs = append(cmdArgs, "--namespace", options.Namespace) } cmdArgs = append(cmdArgs, args...) - command := shell.Command{ + command := helpers.Command{ Command: "kubectl", Args: cmdArgs, Env: options.Env, @@ -56,13 +56,13 @@ func RunKubectlAndGetOutputWithLoggerE(t *testing.T, options *k8s.KubectlOptions } counter := &retry.Counter{ - Count: 3, + Count: 10, Wait: 1 * time.Second, } var output string var err error retry.RunWith(counter, t, func(r *retry.R) { - output, err = shell.RunCommandAndGetOutputE(t, command) + output, err = helpers.RunCommand(r, options, command) if err != nil { // Want to retry on errors connecting to actual Kube API because // these are intermittent. diff --git a/acceptance/framework/logger/logger.go b/acceptance/framework/logger/logger.go index 26777c4e22..34bc0e8a6c 100644 --- a/acceptance/framework/logger/logger.go +++ b/acceptance/framework/logger/logger.go @@ -5,20 +5,19 @@ package logger import ( "fmt" - "testing" + "github.com/hashicorp/consul/sdk/testutil" "time" - terratestTesting "github.com/gruntwork-io/terratest/modules/testing" + terratesting "github.com/gruntwork-io/terratest/modules/testing" ) -// TestLogger implements terratest's TestLogger interface -// so that we can pass it to terratest objects to have consistent logging -// across all tests. +// TestLogger implements Terratest's TestLogger interface so that we can pass it to Terratest objects to have consistent +// logging across all tests. type TestLogger struct{} // Logf takes a format string and args and calls Logf function. -func (tl TestLogger) Logf(t terratestTesting.TestingT, format string, args ...interface{}) { - tt, ok := t.(*testing.T) +func (tl TestLogger) Logf(t terratesting.TestingT, format string, args ...any) { + tt, ok := t.(testutil.TestingTB) if !ok { t.Error("failed to cast") } @@ -27,20 +26,18 @@ func (tl TestLogger) Logf(t terratestTesting.TestingT, format string, args ...in Logf(tt, format, args...) } -// Logf takes a format string and args and logs -// formatted string with a timestamp. -func Logf(t *testing.T, format string, args ...interface{}) { +// Logf takes a format string and args and logs formatted string with a timestamp. +func Logf(t testutil.TestingTB, format string, args ...any) { t.Helper() log := fmt.Sprintf(format, args...) Log(t, log) } -// Log calls t.Log, adding an RFC3339 timestamp to the beginning of the log line. -func Log(t *testing.T, args ...interface{}) { +// Log calls t.Log or r.Log, adding an RFC3339 timestamp to the beginning of the log line. +func Log(t testutil.TestingTB, args ...any) { t.Helper() - - allArgs := []interface{}{time.Now().Format(time.RFC3339)} + allArgs := []any{time.Now().Format(time.RFC3339)} allArgs = append(allArgs, args...) t.Log(allArgs...) } diff --git a/acceptance/go.mod b/acceptance/go.mod index 38b6acdcb0..8bfeac6336 100644 --- a/acceptance/go.mod +++ b/acceptance/go.mod @@ -7,7 +7,7 @@ require ( github.com/gruntwork-io/terratest v0.31.2 github.com/hashicorp/consul-k8s/control-plane v0.0.0-20221117191905-0b1cc2b631e3 github.com/hashicorp/consul/api v1.21.1 - github.com/hashicorp/consul/sdk v0.13.1 + github.com/hashicorp/consul/sdk v0.16.0 github.com/hashicorp/go-uuid v1.0.3 github.com/hashicorp/go-version v1.6.0 github.com/hashicorp/hcp-sdk-go v0.74.0 @@ -30,10 +30,10 @@ require ( github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect github.com/cenkalti/backoff/v3 v3.0.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.10.1 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect - github.com/fatih/color v1.13.0 // indirect + github.com/fatih/color v1.14.1 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-errors/errors v1.0.2-0.20180813162953-d98b870cc4e0 // indirect github.com/go-logr/logr v1.3.0 // indirect @@ -60,7 +60,7 @@ require ( github.com/gruntwork-io/gruntwork-cli v0.7.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-hclog v1.2.2 // indirect + github.com/hashicorp/go-hclog v1.5.0 // indirect github.com/hashicorp/go-immutable-radix v1.3.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-plugin v1.0.1 // indirect @@ -97,7 +97,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pierrec/lz4 v2.5.2+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/pquerna/otp v1.2.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect diff --git a/acceptance/go.sum b/acceptance/go.sum index fc6f6d3be2..3d943aa1cd 100644 --- a/acceptance/go.sum +++ b/acceptance/go.sum @@ -131,8 +131,9 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= @@ -169,8 +170,9 @@ github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCv github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= +github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= @@ -336,8 +338,8 @@ github.com/hashicorp/consul-k8s/control-plane v0.0.0-20221117191905-0b1cc2b631e3 github.com/hashicorp/consul-k8s/control-plane v0.0.0-20221117191905-0b1cc2b631e3/go.mod h1:j9Db/whkzvNC+KP2GftY0HxxleLm9swxXjlu3tYaOAw= github.com/hashicorp/consul/api v1.21.1 h1:gtRiRHuH1CZuQFlhvSjRiih3Yjufb99nbq1hOY/kaug= github.com/hashicorp/consul/api v1.21.1/go.mod h1:5LfOMDAWm5h53/5RBneTJBopM25YBL9+ihyDHYJJhNg= -github.com/hashicorp/consul/sdk v0.13.1 h1:EygWVWWMczTzXGpO93awkHFzfUka6hLYJ0qhETd+6lY= -github.com/hashicorp/consul/sdk v0.13.1/go.mod h1:SW/mM4LbKfqmMvcFu8v+eiQQ7oitXEFeiBe9StxERb0= +github.com/hashicorp/consul/sdk v0.16.0 h1:SE9m0W6DEfgIVCJX7xU+iv/hUl4m/nxqMTnCdMxDpJ8= +github.com/hashicorp/consul/sdk v0.16.0/go.mod h1:7pxqqhqoaPqnBnzXD1StKed62LqJeClzVsUEy85Zr0A= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -350,8 +352,8 @@ github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrj github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v1.2.2 h1:ihRI7YFwcZdiSD7SIenIhHfQH3OuDvWerAUBZbeQS3M= -github.com/hashicorp/go-hclog v1.2.2/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= +github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.1.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.0 h1:8exGP7ego3OmkfksihtSouGMZ+hQrhxx+FVELeXpVPE= @@ -573,8 +575,9 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=