From 37a492ac25cab36454b190341fe4cbefd76b3585 Mon Sep 17 00:00:00 2001 From: magodo Date: Tue, 21 Jun 2022 10:55:22 +0800 Subject: [PATCH] New option: `--append` to allow importing to existing workspace (#151) For local backend, `aztfy` will by default ensure the output directory is empty at the very begining. This is to avoid any conflicts happen for existing user files, including the terraform configuration, provider configuration, the state file, etc. As a result, `aztfy` generates a pretty new workspace for users. One limitation of doing so is users can't import resources to existing state file via `aztfy`. To support this scenario, this PR adds a new option `--append`. This option will make `aztfy` skip the empty guarantee for the output directory. If the output directory is empty, then it has no effect. Otherwise, it will ensure the provider setting (create a file for it if not exists). Then it proceeds the following steps. This means if the output directory has an active Terraform workspace, i.e. there exists a state file, any resource imported by the `aztfy` will be imported into that state file. Especially, the file generated by `aztfy` in this case will be named differently than normal, where each file will has `.aztfy` suffix before the extension (e.g. `main.aztfy.tf`), to avoid potential file name conflicts. --- README.md | 8 ++ internal/config/config.go | 1 + internal/meta/meta_impl.go | 207 +++++++++++++++++++++++++++-------- internal/test/append_test.go | 105 ++++++++++++++++++ internal/test/e2e_test.go | 2 +- main.go | 16 +++ 6 files changed, 294 insertions(+), 45 deletions(-) create mode 100644 internal/test/append_test.go diff --git a/README.md b/README.md index 96c8887..cf79347 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,14 @@ E.g. to use the [`azurerm` backend](https://www.terraform.io/language/settings/b aztfy --backend-type=azurerm --backend-config=resource_group_name= --backend-config=storage_account_name= --backend-config=container_name= --backend-config=key=terraform.tfstate ``` +### Import Into Existing Local State + +For local backend, `aztfy` will by default ensure the output directory is empty at the very begining. This is to avoid any conflicts happen for existing user files, including the terraform configuration, provider configuration, the state file, etc. As a result, `aztfy` generates a pretty new workspace for users. + +One limitation of doing so is users can't import resources to existing state file via `aztfy`. To support this scenario, you can use the `--append` option. This option will make `aztfy` skip the empty guarantee for the output directory. If the output directory is empty, then it has no effect. Otherwise, it will ensure the provider setting (create a file for it if not exists). Then it proceeds the following steps. + +This means if the output directory has an active Terraform workspace, i.e. there exists a state file, any resource imported by the `aztfy` will be imported into that state file. Especially, the file generated by `aztfy` in this case will be named differently than normal, where each file will has `.aztfy` suffix before the extension (e.g. `main.aztfy.tf`), to avoid potential file name conflicts. If you run `aztfy --append` multiple times, the generated config in `main.aztfy.tf` will be appended in each run. + ## Demo [![asciicast](https://asciinema.org/a/475516.svg)](https://asciinema.org/a/475516) diff --git a/internal/config/config.go b/internal/config/config.go index 4688b87..299d78a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -11,6 +11,7 @@ type Config struct { OutputDir string ResourceNamePattern string Overwrite bool + Append bool BatchMode bool BackendType string BackendConfig []string diff --git a/internal/meta/meta_impl.go b/internal/meta/meta_impl.go index 1f49b64..163827e 100644 --- a/internal/meta/meta_impl.go +++ b/internal/meta/meta_impl.go @@ -1,6 +1,7 @@ package meta import ( + "bufio" "bytes" "context" "encoding/json" @@ -8,6 +9,7 @@ import ( "io" "os" "path/filepath" + "regexp" "strings" "time" @@ -46,6 +48,10 @@ type MetaImpl struct { backendType string backendConfig []string + + // Use a safer name which is less likely to conflicts with users' existing files. + // This is mainly used for the --append option. + useSafeFilename bool } func newMetaImpl(cfg config.Config) (Meta, error) { @@ -71,39 +77,32 @@ func newMetaImpl(cfg config.Config) (Meta, error) { return nil, err } } - stat, err := os.Stat(outdir) - if os.IsNotExist(err) { - return nil, fmt.Errorf("the output directory %q doesn't exist", outdir) - } - if !stat.IsDir() { - return nil, fmt.Errorf("the output path %q is not a directory", outdir) - } - dir, err := os.Open(outdir) + empty, err := dirIsEmpty(outdir) if err != nil { return nil, err } - _, err = dir.Readdirnames(1) - dir.Close() - if err != io.EOF { - if cfg.Overwrite { - if err := removeEverythingUnder(outdir); err != nil { - return nil, err - } - } else { - if cfg.BatchMode { - return nil, fmt.Errorf("the output directory %q is not empty", outdir) - } - - // Interactive mode - fmt.Printf("The output directory is not empty - overwrite (Y/N)? ") - var ans string - fmt.Scanf("%s", &ans) - if !strings.EqualFold(ans, "y") { - return nil, fmt.Errorf("the output directory %q is not empty", outdir) - } else { + if !empty { + if !cfg.Append { + if cfg.Overwrite { if err := removeEverythingUnder(outdir); err != nil { return nil, err } + } else { + if cfg.BatchMode { + return nil, fmt.Errorf("the output directory %q is not empty", outdir) + } + + // Interactive mode + fmt.Printf("The output directory is not empty - overwrite (Y/N)? ") + var ans string + fmt.Scanf("%s", &ans) + if !strings.EqualFold(ans, "y") { + return nil, fmt.Errorf("the output directory %q is not empty", outdir) + } else { + if err := removeEverythingUnder(outdir); err != nil { + return nil, err + } + } } } } @@ -126,6 +125,7 @@ func newMetaImpl(cfg config.Config) (Meta, error) { resourceMapping: cfg.ResourceMapping, backendType: cfg.BackendType, backendConfig: cfg.BackendConfig, + useSafeFilename: cfg.Append, } if pos := strings.LastIndex(cfg.ResourceNamePattern, "*"); pos != -1 { @@ -186,7 +186,6 @@ func (meta *MetaImpl) ListResource() (ImportList, error) { var l ImportList - // No resource mapping specified, simply insert each listed id to the import list. for i, id := range ids { recommendations := RecommendationsForId(id) item := ImportItem{ @@ -224,7 +223,7 @@ func (meta MetaImpl) Import(item *ImportItem) { // Generate a temp Terraform config to include the empty template for each resource. // This is required for the following importing. - cfgFile := filepath.Join(meta.outdir, "main.tf") + cfgFile := filepath.Join(meta.outdir, meta.filenameTmpCfg()) tpl := fmt.Sprintf(`resource "%s" "%s" {}`, item.TFAddr.Type, item.TFAddr.Name) if err := os.WriteFile(cfgFile, []byte(tpl), 0644); err != nil { item.ImportError = fmt.Errorf("generating resource template file: %w", err) @@ -286,20 +285,61 @@ provider "azurerm" { `, meta.backendType, azurerm.ProviderSchemaInfo.Version) } +func (meta MetaImpl) filenameProviderSetting() string { + if meta.useSafeFilename { + return "provider.aztfy.tf" + } + return "provider.tf" +} + +func (meta MetaImpl) filenameMainCfg() string { + if meta.useSafeFilename { + return "main.aztfy.tf" + } + return "main.tf" +} + +func (meta MetaImpl) filenameTmpCfg() string { + return "tmp.aztfy.tf" +} + func (meta *MetaImpl) initProvider(ctx context.Context) error { - cfgFile := filepath.Join(meta.outdir, "provider.tf") + empty, err := dirIsEmpty(meta.outdir) + if err != nil { + return err + } - // Always use the latest provider version here, as this is a one shot tool, which should guarantees to work with the latest version. - if err := os.WriteFile(cfgFile, []byte(meta.providerConfig()), 0644); err != nil { - return fmt.Errorf("error creating provider config: %w", err) + // If the directory is empty, generate the full config as the output directory is empty. + // Otherwise: + // - If the output directory already exists the `azurerm` provider setting, then do nothing + // - Otherwise, just generate the `azurerm` provider setting (as it is only for local backend) + if empty { + cfgFile := filepath.Join(meta.outdir, meta.filenameProviderSetting()) + if err := os.WriteFile(cfgFile, []byte(meta.providerConfig()), 0644); err != nil { + return fmt.Errorf("error creating provider config: %w", err) + } + + var opts []tfexec.InitOption + for _, opt := range meta.backendConfig { + opts = append(opts, tfexec.BackendConfig(opt)) + } + if err := meta.tf.Init(ctx, opts...); err != nil { + return fmt.Errorf("error running terraform init: %s", err) + } + return nil } - var opts []tfexec.InitOption - for _, opt := range meta.backendConfig { - opts = append(opts, tfexec.BackendConfig(opt)) + exists, err := dirContainsProviderSetting(meta.outdir) + if err != nil { + return err } - if err := meta.tf.Init(ctx, opts...); err != nil { - return fmt.Errorf("error running terraform init: %s", err) + if !exists { + if err := appendToFile(meta.filenameProviderSetting(), `provider "azurerm" { + features {} +} +`); err != nil { + return fmt.Errorf("error creating provider config: %w", err) + } } return nil @@ -418,17 +458,15 @@ func (meta MetaImpl) resolveDependency(configs ConfigInfos) (ConfigInfos, error) } func (meta MetaImpl) generateConfig(cfgs ConfigInfos) error { - cfgFile := filepath.Join(meta.outdir, "main.tf") + cfgFile := filepath.Join(meta.outdir, meta.filenameMainCfg()) buf := bytes.NewBuffer([]byte{}) - for i, cfg := range cfgs { + for _, cfg := range cfgs { if _, err := cfg.DumpHCL(buf); err != nil { return err } - if i != len(cfgs)-1 { - buf.Write([]byte("\n")) - } + buf.Write([]byte("\n")) } - if err := os.WriteFile(cfgFile, buf.Bytes(), 0644); err != nil { + if err := appendToFile(cfgFile, buf.String()); err != nil { return fmt.Errorf("generating main configuration file: %w", err) } @@ -465,3 +503,84 @@ func removeEverythingUnder(path string) error { dir.Close() return nil } + +func dirIsEmpty(path string) (bool, error) { + stat, err := os.Stat(path) + if os.IsNotExist(err) { + return false, fmt.Errorf("the path %q doesn't exist", path) + } + if !stat.IsDir() { + return false, fmt.Errorf("the path %q is not a directory", path) + } + dir, err := os.Open(path) + if err != nil { + return false, err + } + _, err = dir.Readdirnames(1) + if err != nil { + if err == io.EOF { + dir.Close() + return true, nil + } + return false, err + } + dir.Close() + return false, nil +} + +func dirContainsProviderSetting(path string) (bool, error) { + stat, err := os.Stat(path) + if os.IsNotExist(err) { + return false, fmt.Errorf("the path %q doesn't exist", path) + } + if !stat.IsDir() { + return false, fmt.Errorf("the path %q is not a directory", path) + } + dir, err := os.Open(path) + if err != nil { + return false, err + } + defer dir.Close() + + fnames, err := dir.Readdirnames(0) + if err != nil { + return false, err + } + + // Ideally, we shall use hclgrep for a perfect match. But as the provider setting is simple enough, we do a text matching here. + p := regexp.MustCompile(`^\s*provider\s+"azurerm"\s*{\s*$`) + for _, fname := range fnames { + // fmt.Println(fname) + // fmt.Println(filepath.Ext(fname)) + if filepath.Ext(fname) != ".tf" { + continue + } + f, err := os.Open(filepath.Join(path, fname)) + if err != nil { + return false, fmt.Errorf("opening %s: %v", fname, err) + } + scanner := bufio.NewScanner(f) + for scanner.Scan() { + if p.MatchString(scanner.Text()) { + f.Close() + return true, nil + } + } + if err := scanner.Err(); err != nil { + f.Close() + return false, fmt.Errorf("reading file %s: %v", fname, err) + } + f.Close() + } + return false, nil +} + +func appendToFile(path, content string) error { + f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer f.Close() + _, err = f.WriteString(content) + return err +} diff --git a/internal/test/append_test.go b/internal/test/append_test.go new file mode 100644 index 0000000..59e8826 --- /dev/null +++ b/internal/test/append_test.go @@ -0,0 +1,105 @@ +package test + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/Azure/aztfy/internal" + "github.com/Azure/aztfy/internal/config" + "github.com/hashicorp/terraform-exec/tfexec" +) + +func TestAppendMode(t *testing.T) { + t.Parallel() + precheck(t) + d := NewData() + tfexecPath := ensureTF(t) + provisionDir := t.TempDir() + os.Chdir(provisionDir) + if err := os.WriteFile("main.tf", []byte(fmt.Sprintf(` +provider "azurerm" { + features {} +} +resource "azurerm_resource_group" "test1" { + name = "%[1]s1" + location = "WestEurope" +} +resource "azurerm_resource_group" "test2" { + name = "%[1]s2" + location = "WestEurope" +} +resource "azurerm_resource_group" "test3" { + name = "%[1]s3" + location = "WestEurope" +} +`, d.RandomRgName())), 0644); err != nil { + t.Fatalf("created to create the TF config file: %v", err) + } + tf, err := tfexec.NewTerraform(provisionDir, tfexecPath) + if err != nil { + t.Fatalf("failed to new terraform: %v", err) + } + ctx := context.Background() + if err := tf.Init(ctx); err != nil { + t.Fatalf("terraform init failed: %v", err) + } + if err := tf.Apply(ctx); err != nil { + t.Fatalf("terraform apply failed: %v", err) + } + defer func() { + if err := tf.Destroy(ctx); err != nil { + t.Logf("terraform destroy failed: %v", err) + } + }() + + // Import the first resource group + aztfyDir := t.TempDir() + cfg := config.Config{ + SubscriptionId: os.Getenv("ARM_SUBSCRIPTION_ID"), + OutputDir: aztfyDir, + BackendType: "local", + ResourceNamePattern: "t1", + } + cfg.ResourceGroupName = d.RandomRgName() + "1" + cfg.ResourceNamePattern = "round1_" + if err := internal.BatchImport(cfg, false); err != nil { + t.Fatalf("failed to run first batch import: %v", err) + } + // Import the second resource group mutably + cfg.Append = true + cfg.ResourceGroupName = d.RandomRgName() + "2" + cfg.ResourceNamePattern = "round2_" + if err := internal.BatchImport(cfg, false); err != nil { + t.Fatalf("failed to run second batch import: %v", err) + } + // Import the third resource group mutably + cfg.Append = true + cfg.ResourceGroupName = d.RandomRgName() + "3" + cfg.ResourceNamePattern = "round3_" + if err := internal.BatchImport(cfg, false); err != nil { + t.Fatalf("failed to run second batch import: %v", err) + } + + // Verify + tf2, err := tfexec.NewTerraform(aztfyDir, tfexecPath) + if err != nil { + t.Fatalf("failed to new terraform: %v", err) + } + diff, err := tf2.Plan(ctx) + if err != nil { + t.Fatalf("terraform plan in the generated workspace failed: %v", err) + } + if diff { + t.Fatalf("terraform plan shows diff") + } + state, err := tf2.ShowStateFile(ctx, filepath.Join(aztfyDir, "terraform.tfstate")) + if err != nil { + t.Fatalf("terraform state show in the generated workspace failed: %v", err) + } + if n := len(state.Values.RootModule.Resources); n != 3 { + t.Fatalf("expected terrafied resource: %d, got=%d", 3, n) + } +} diff --git a/internal/test/e2e_test.go b/internal/test/e2e_test.go index 24e9368..0e1bb31 100644 --- a/internal/test/e2e_test.go +++ b/internal/test/e2e_test.go @@ -104,7 +104,7 @@ func runCase(t *testing.T, d Data, c Case) { t.Fatalf("terraform state show in the generated workspace failed: %v", err) } if n, expect := len(state.Values.RootModule.Resources), len(resourceMapping); n != expect { - t.Fatalf("expected terrified resource: %d, got=%d", expect, n) + t.Fatalf("expected terrafied resource: %d, got=%d", expect, n) } } diff --git a/main.go b/main.go index 3c49dbe..95115e4 100644 --- a/main.go +++ b/main.go @@ -28,6 +28,7 @@ func main() { flagContinue bool flagPattern string flagOverwrite bool + flagAppend bool flagBackendType string flagBackendConfig cli.StringSlice @@ -96,6 +97,12 @@ func main() { Usage: "Whether to overwrite the output directory if it is not empty (use with caution)", Destination: &flagOverwrite, }, + &cli.BoolFlag{ + Name: "append", + EnvVars: []string{"AZTFY_APPEND"}, + Usage: "Skip cleaning up the output directory prior to importing, everything will be imported to the existing state file if any (local backend only)", + Destination: &flagAppend, + }, &cli.StringFlag{ Name: "backend-type", EnvVars: []string{"AZTFY_BACKEND_TYPE"}, @@ -140,6 +147,14 @@ func main() { if flagContinue && !flagBatchMode { return fmt.Errorf("`--continue` must be used together with `--batch`") } + if flagAppend { + if flagBackendType != "local" { + return fmt.Errorf("`--append` only works for local backend") + } + if flagOverwrite { + return fmt.Errorf("`--append` conflicts with `--overwrite`") + } + } rg := c.Args().First() @@ -192,6 +207,7 @@ func main() { cfg.OutputDir = flagOutputDir cfg.ResourceNamePattern = flagPattern cfg.Overwrite = flagOverwrite + cfg.Append = flagAppend cfg.BatchMode = flagBatchMode cfg.BackendType = flagBackendType cfg.BackendConfig = flagBackendConfig.Value()