New option: --append to allow importing to existing workspace (#151)

For local backend, `aztfy` will by default ensure the output directory is empty at the very begining. This is to avoid any conflicts happen for existing user files, including the terraform configuration, provider configuration, the state file, etc. As a result, `aztfy` generates a pretty new workspace for users.

One limitation of doing so is users can't import resources to existing state file via `aztfy`. To support this scenario, this PR adds a new option `--append`. This option will make `aztfy` skip the empty guarantee for the output directory. If the output directory is empty, then it has no effect. Otherwise, it will ensure the provider setting (create a file for it if not exists). Then it proceeds the following steps.

This means if the output directory has an active Terraform workspace, i.e. there exists a state file, any resource imported by the `aztfy` will be imported into that state file.  Especially, the file generated by `aztfy` in this case will be named differently than normal, where each file will has `.aztfy` suffix before the extension (e.g. `main.aztfy.tf`), to avoid potential file name conflicts.

README.md

@@ -90,6 +90,14 @@ E.g. to use the [`azurerm` backend](https://www.terraform.io/language/settings/b
 aztfy --backend-type=azurerm --backend-config=resource_group_name=<resource group name> --backend-config=storage_account_name=<account name> --backend-config=container_name=<container name> --backend-config=key=terraform.tfstate <importing resource group name>
 ```
 
+### Import Into Existing Local State
+
+For local backend, `aztfy` will by default ensure the output directory is empty at the very begining. This is to avoid any conflicts happen for existing user files, including the terraform configuration, provider configuration, the state file, etc. As a result, `aztfy` generates a pretty new workspace for users.
+
+One limitation of doing so is users can't import resources to existing state file via `aztfy`. To support this scenario, you can use the `--append` option. This option will make `aztfy` skip the empty guarantee for the output directory. If the output directory is empty, then it has no effect. Otherwise, it will ensure the provider setting (create a file for it if not exists). Then it proceeds the following steps.
+
+This means if the output directory has an active Terraform workspace, i.e. there exists a state file, any resource imported by the `aztfy` will be imported into that state file. Especially, the file generated by `aztfy` in this case will be named differently than normal, where each file will has `.aztfy` suffix before the extension (e.g. `main.aztfy.tf`), to avoid potential file name conflicts. If you run `aztfy --append` multiple times, the generated config in `main.aztfy.tf` will be appended in each run.
+
 ## Demo
 
 [![asciicast](https://asciinema.org/a/475516.svg)](https://asciinema.org/a/475516)

internal/config/config.go

@@ -11,6 +11,7 @@ type Config struct {
 	OutputDir           string
 	ResourceNamePattern string
 	Overwrite           bool
+	Append              bool
 	BatchMode           bool
 	BackendType         string
 	BackendConfig       []string

internal/meta/meta_impl.go

@@ -1,13 +1,15 @@
 package meta
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"time"
 
@@ -46,6 +48,10 @@ type MetaImpl struct {
 
 	backendType   string
 	backendConfig []string
+
+	// Use a safer name which is less likely to conflicts with users' existing files.
+	// This is mainly used for the --append option.
+	useSafeFilename bool
 }
 
 func newMetaImpl(cfg config.Config) (Meta, error) {
@@ -71,39 +77,32 @@ func newMetaImpl(cfg config.Config) (Meta, error) {
 			return nil, err
 		}
 	}
-	stat, err := os.Stat(outdir)
-	if os.IsNotExist(err) {
-		return nil, fmt.Errorf("the output directory %q doesn't exist", outdir)
-	}
-	if !stat.IsDir() {
-		return nil, fmt.Errorf("the output path %q is not a directory", outdir)
-	}
-	dir, err := os.Open(outdir)
+	empty, err := dirIsEmpty(outdir)
 	if err != nil {
 		return nil, err
 	}
-	_, err = dir.Readdirnames(1)
-	dir.Close()
-	if err != io.EOF {
-		if cfg.Overwrite {
-			if err := removeEverythingUnder(outdir); err != nil {
-				return nil, err
-			}
-		} else {
-			if cfg.BatchMode {
-				return nil, fmt.Errorf("the output directory %q is not empty", outdir)
-			}
-
-			// Interactive mode
-			fmt.Printf("The output directory is not empty - overwrite (Y/N)? ")
-			var ans string
-			fmt.Scanf("%s", &ans)
-			if !strings.EqualFold(ans, "y") {
-				return nil, fmt.Errorf("the output directory %q is not empty", outdir)
-			} else {
+	if !empty {
+		if !cfg.Append {
+			if cfg.Overwrite {
 				if err := removeEverythingUnder(outdir); err != nil {
 					return nil, err
 				}
+			} else {
+				if cfg.BatchMode {
+					return nil, fmt.Errorf("the output directory %q is not empty", outdir)
+				}
+
+				// Interactive mode
+				fmt.Printf("The output directory is not empty - overwrite (Y/N)? ")
+				var ans string
+				fmt.Scanf("%s", &ans)
+				if !strings.EqualFold(ans, "y") {
+					return nil, fmt.Errorf("the output directory %q is not empty", outdir)
+				} else {
+					if err := removeEverythingUnder(outdir); err != nil {
+						return nil, err
+					}
+				}
 			}
 		}
 	}
@@ -126,6 +125,7 @@ func newMetaImpl(cfg config.Config) (Meta, error) {
 		resourceMapping: cfg.ResourceMapping,
 		backendType:     cfg.BackendType,
 		backendConfig:   cfg.BackendConfig,
+		useSafeFilename: cfg.Append,
 	}
 
 	if pos := strings.LastIndex(cfg.ResourceNamePattern, "*"); pos != -1 {
@@ -186,7 +186,6 @@ func (meta *MetaImpl) ListResource() (ImportList, error) {
 
 	var l ImportList
 
-	// No resource mapping specified, simply insert each listed id to the import list.
 	for i, id := range ids {
 		recommendations := RecommendationsForId(id)
 		item := ImportItem{
@@ -224,7 +223,7 @@ func (meta MetaImpl) Import(item *ImportItem) {
 
 	// Generate a temp Terraform config to include the empty template for each resource.
 	// This is required for the following importing.
-	cfgFile := filepath.Join(meta.outdir, "main.tf")
+	cfgFile := filepath.Join(meta.outdir, meta.filenameTmpCfg())
 	tpl := fmt.Sprintf(`resource "%s" "%s" {}`, item.TFAddr.Type, item.TFAddr.Name)
 	if err := os.WriteFile(cfgFile, []byte(tpl), 0644); err != nil {
 		item.ImportError = fmt.Errorf("generating resource template file: %w", err)
@@ -286,20 +285,61 @@ provider "azurerm" {
 `, meta.backendType, azurerm.ProviderSchemaInfo.Version)
 }
 
+func (meta MetaImpl) filenameProviderSetting() string {
+	if meta.useSafeFilename {
+		return "provider.aztfy.tf"
+	}
+	return "provider.tf"
+}
+
+func (meta MetaImpl) filenameMainCfg() string {
+	if meta.useSafeFilename {
+		return "main.aztfy.tf"
+	}
+	return "main.tf"
+}
+
+func (meta MetaImpl) filenameTmpCfg() string {
+	return "tmp.aztfy.tf"
+}
+
 func (meta *MetaImpl) initProvider(ctx context.Context) error {
-	cfgFile := filepath.Join(meta.outdir, "provider.tf")
+	empty, err := dirIsEmpty(meta.outdir)
+	if err != nil {
+		return err
+	}
 
-	// Always use the latest provider version here, as this is a one shot tool, which should guarantees to work with the latest version.
-	if err := os.WriteFile(cfgFile, []byte(meta.providerConfig()), 0644); err != nil {
-		return fmt.Errorf("error creating provider config: %w", err)
+	// If the directory is empty, generate the full config as the output directory is empty.
+	// Otherwise:
+	// - If the output directory already exists the `azurerm` provider setting, then do nothing
+	// - Otherwise, just generate the `azurerm` provider setting (as it is only for local backend)
+	if empty {
+		cfgFile := filepath.Join(meta.outdir, meta.filenameProviderSetting())
+		if err := os.WriteFile(cfgFile, []byte(meta.providerConfig()), 0644); err != nil {
+			return fmt.Errorf("error creating provider config: %w", err)
+		}
+
+		var opts []tfexec.InitOption
+		for _, opt := range meta.backendConfig {
+			opts = append(opts, tfexec.BackendConfig(opt))
+		}
+		if err := meta.tf.Init(ctx, opts...); err != nil {
+			return fmt.Errorf("error running terraform init: %s", err)
+		}
+		return nil
 	}
 
-	var opts []tfexec.InitOption
-	for _, opt := range meta.backendConfig {
-		opts = append(opts, tfexec.BackendConfig(opt))
+	exists, err := dirContainsProviderSetting(meta.outdir)
+	if err != nil {
+		return err
 	}
-	if err := meta.tf.Init(ctx, opts...); err != nil {
-		return fmt.Errorf("error running terraform init: %s", err)
+	if !exists {
+		if err := appendToFile(meta.filenameProviderSetting(), `provider "azurerm" {
+  features {}
+}
+`); err != nil {
+			return fmt.Errorf("error creating provider config: %w", err)
+		}
 	}
 
 	return nil
@@ -418,17 +458,15 @@ func (meta MetaImpl) resolveDependency(configs ConfigInfos) (ConfigInfos, error)
 }
 
 func (meta MetaImpl) generateConfig(cfgs ConfigInfos) error {
-	cfgFile := filepath.Join(meta.outdir, "main.tf")
+	cfgFile := filepath.Join(meta.outdir, meta.filenameMainCfg())
 	buf := bytes.NewBuffer([]byte{})
-	for i, cfg := range cfgs {
+	for _, cfg := range cfgs {
 		if _, err := cfg.DumpHCL(buf); err != nil {
 			return err
 		}
-		if i != len(cfgs)-1 {
-			buf.Write([]byte("\n"))
-		}
+		buf.Write([]byte("\n"))
 	}
-	if err := os.WriteFile(cfgFile, buf.Bytes(), 0644); err != nil {
+	if err := appendToFile(cfgFile, buf.String()); err != nil {
 		return fmt.Errorf("generating main configuration file: %w", err)
 	}
 
@@ -465,3 +503,84 @@ func removeEverythingUnder(path string) error {
 	dir.Close()
 	return nil
 }
+
+func dirIsEmpty(path string) (bool, error) {
+	stat, err := os.Stat(path)
+	if os.IsNotExist(err) {
+		return false, fmt.Errorf("the path %q doesn't exist", path)
+	}
+	if !stat.IsDir() {
+		return false, fmt.Errorf("the path %q is not a directory", path)
+	}
+	dir, err := os.Open(path)
+	if err != nil {
+		return false, err
+	}
+	_, err = dir.Readdirnames(1)
+	if err != nil {
+		if err == io.EOF {
+			dir.Close()
+			return true, nil
+		}
+		return false, err
+	}
+	dir.Close()
+	return false, nil
+}
+
+func dirContainsProviderSetting(path string) (bool, error) {
+	stat, err := os.Stat(path)
+	if os.IsNotExist(err) {
+		return false, fmt.Errorf("the path %q doesn't exist", path)
+	}
+	if !stat.IsDir() {
+		return false, fmt.Errorf("the path %q is not a directory", path)
+	}
+	dir, err := os.Open(path)
+	if err != nil {
+		return false, err
+	}
+	defer dir.Close()
+
+	fnames, err := dir.Readdirnames(0)
+	if err != nil {
+		return false, err
+	}
+
+	// Ideally, we shall use hclgrep for a perfect match. But as the provider setting is simple enough, we do a text matching here.
+	p := regexp.MustCompile(`^\s*provider\s+"azurerm"\s*{\s*$`)
+	for _, fname := range fnames {
+		// fmt.Println(fname)
+		// fmt.Println(filepath.Ext(fname))
+		if filepath.Ext(fname) != ".tf" {
+			continue
+		}
+		f, err := os.Open(filepath.Join(path, fname))
+		if err != nil {
+			return false, fmt.Errorf("opening %s: %v", fname, err)
+		}
+		scanner := bufio.NewScanner(f)
+		for scanner.Scan() {
+			if p.MatchString(scanner.Text()) {
+				f.Close()
+				return true, nil
+			}
+		}
+		if err := scanner.Err(); err != nil {
+			f.Close()
+			return false, fmt.Errorf("reading file %s: %v", fname, err)
+		}
+		f.Close()
+	}
+	return false, nil
+}
+
+func appendToFile(path, content string) error {
+	f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	_, err = f.WriteString(content)
+	return err
+}