pkg/asset: Introduce Load() into the Asset interface that loads assets (from disk)

cmd/openshift-install/targets.go

@@ -68,18 +68,18 @@ func newTargetsCmd() []*cobra.Command {
 
 func runTargetCmd(targets ...asset.WritableAsset) func(cmd *cobra.Command, args []string) error {
 	return func(cmd *cobra.Command, args []string) error {
-		assetStore := &asset.StoreImpl{}
-		err := assetStore.Load(rootOpts.dir)
+		assetStore, err := asset.NewStore(rootOpts.dir)
 		if err != nil {
-			logrus.Errorf("Could not load assets from state file: %v", err)
+			return errors.Wrapf(err, "failed to create asset store")
 		}
+
 		for _, a := range targets {
 			err := assetStore.Fetch(a)
 			if err != nil {
 				if exitError, ok := errors.Cause(err).(*exec.ExitError); ok && len(exitError.Stderr) > 0 {
 					logrus.Error(strings.Trim(string(exitError.Stderr), "\n"))
 				}
-				err = errors.Wrapf(err, "failed to generate %s", a.Name())
+				err = errors.Wrapf(err, "failed to fetch %s", a.Name())
 			}
 
 			if err2 := asset.PersistToFile(a, rootOpts.dir); err2 != nil {
@@ -95,11 +95,15 @@ func runTargetCmd(targets ...asset.WritableAsset) func(cmd *cobra.Command, args
 				return err
 			}
 		}
-		err = assetStore.Save(rootOpts.dir)
-		if err != nil {
-			errors.Wrapf(err, "failed to write to state file")
-			return err
+
+		if err := assetStore.Save(rootOpts.dir); err != nil {
+			return errors.Wrapf(err, "failed to write to state file")
 		}
+
+		if err := assetStore.Purge(targets); err != nil {
+			return errors.Wrapf(err, "failed to delete existing on-disk files")
+		}
+
 		return nil
 	}
 }

docs/design/assetgeneration.md

@@ -1,4 +1,4 @@
-# Asset generation
+# Asset Generation
 
 The installer internally uses a directed acyclic graph to represent all of the assets it creates as well as their dependencies. This process looks very similar to many build systems (e.g. Bazel, Make).
 
@@ -16,43 +16,67 @@ An asset is the generic representation of work-item for installer that needs to
 
 The asset would usually follow these steps to generate its output:
 
-1. Load components from disk. (If found, mark them from deletion, we consume our inputs.)
+1. Fetch its parent assets.
 
-2. Load its components from State object given by installer.
+2. Generate the assets either by:
+    *  Using the parent assets
+    *  Loading from on-disk assets
+    *  Loading from state file
 
-3. If both the sources exist,
+3. If any of the parent assets are **dirty** (currently we think all on-disk assets are **dirty**), then use the parent assets to generate and return **dirty**.
 
-    * Both are same: nothing to do use the State object.
+4. If none of the parent assets are **dirty**, but the asset itself is on disk, then use the on-disk asset and return **dirty**.
 
-    * Both differ: maybe try to merge and mark itself **dirty** or error out.
+5. If none of the parent assets or this asset is **dirty**, but the asset is found in the state file, then use the asset from state file and return **NOT dirty**.
 
-4. If only one exists, Move the disk source to State object.
-
-5. If none exists, Use the assets from State object to create the components and store them in the State object.
+6. If none of the parent assets are **dirty**, this asset is not **dirty**, and this asset is not found in the state file, then generate the asset using its parent assets and return **NOT dirty**.
 
 An example of the Asset:
 
 ```go
-type Asset interface{
+type Asset interface {
     Dependencies() []Assets
-    Generate(State) error
+    Generate(Parents) error
+    Name() string
 }
 ```
 
 ## Writable Asset
 
 A writable asset is an asset that generates files to write to disk. These files could be for the user to consume as output from installer targets, such as install-config.yml from the InstallConfig asset. Or these files could be used internally by the installer, such as the cert/key files generated by TLS assets.
+A writable asset can also be loaded from disk to construct.
 
 ```go
 type WritableAsset interface{
     Asset
     Files() []File
+    Load(FileFetcher) (found bool, err error)
 }
 
 type File struct {
     Filename string
     Data []byte
 }
+
+// FileFetcher is passed to every Loadable asset when implementing
+// the Load() function. The FileFetcher enables the Loadable asset
+// to read specific file(s) from disk.
+type FileFetcher interface {
+    // FetchByName returns the file with the given name.
+    FetchByName(string) *File
+    // FetchByPattern returns the files whose name match the given regexp.
+    FetchByPattern(*regexp.Regexp) ([]*File, error)
+}
+```
+After being loaded and consumed by a children asset, the existing on-disk asset will be purged.
+E.g.
+
+```shell
+$ openshift-install install-config
+# Generate install-config.yml
+
+$ openshift-install manifests
+# Generate manifests/ and tectonic/ dir, also remove install-config.yml
 ```
 
 ## Target generation
@@ -61,9 +85,7 @@ The installer uses depth-first traversal on the dependency graph, starting at th
 
 ### Dirty detection
 
-An asset generation reports **DIRTY** when it detects that the components have been modified from previous run.
-
-When generating dependencies of an asset, if any one of the dependencies report dirty, the installer informs the asset that its dependencies are dirty. The asset can either generate from its dependencies or exit with error.
+An asset generation reports **DIRTY** when it detects that the components have been modified from previous run. For now the asset is considered dirty when it's on-disk.
 
 ### Example
 

pkg/asset/asset.go

@@ -21,11 +21,16 @@ type Asset interface {
 }
 
 // WritableAsset is an Asset that has files that can be written to disk.
+// It can also be loaded from disk.
 type WritableAsset interface {
 	Asset
 
 	// Files returns the files to write.
 	Files() []*File
+
+	// Load returns the on-disk asset if it exists.
+	// The asset object should be changed only when it's loaded successfully.
+	Load(FileFetcher) (found bool, err error)
 }
 
 // File is a file for an Asset.

pkg/asset/asset_test.go

@@ -33,6 +33,10 @@ func (a *writablePersistAsset) Files() []*File {
 	return a.FileList
 }
 
+func (a *writablePersistAsset) Load(FileFetcher) (bool, error) {
+	return false, nil
+}
+
 func TestPersistToFile(t *testing.T) {
 	cases := []struct {
 		name      string

pkg/asset/cluster/cluster.go

@@ -22,7 +22,7 @@ const (
 	// MetadataFilename is name of the file where clustermetadata is stored.
 	MetadataFilename = "metadata.json"
 
-	stateFileName = "terraform.state"
+	stateFileName = "terraform.tfstate"
 )
 
 // Cluster uses the terraform executable to launch a cluster
@@ -67,7 +67,7 @@ func (c *Cluster) Generate(parents asset.Parents) (err error) {
 		return errors.Wrap(err, "failed to write terraform.tfvars file")
 	}
 
-	platform := terraformVariables.Platform
+	platform := installConfig.Config.Platform.Name()
 	if err := data.Unpack(tmpDir, platform); err != nil {
 		return err
 	}
@@ -155,3 +155,12 @@ func (c *Cluster) Generate(parents asset.Parents) (err error) {
 func (c *Cluster) Files() []*asset.File {
 	return c.FileList
 }
+
+// Load returns error if the tfstate file is already on-disk, because we want to
+// prevent user from accidentally re-launching the cluster.
+func (c *Cluster) Load(f asset.FileFetcher) (found bool, err error) {
+	if f.FetchByName(stateFileName) != nil {
+		return true, fmt.Errorf("%q already exisits", stateFileName)
+	}
+	return false, nil
+}

pkg/asset/cluster/tfvars.go

@@ -17,8 +17,7 @@ const (
 // TerraformVariables depends on InstallConfig and
 // Ignition to generate the terrafor.tfvars.
 type TerraformVariables struct {
-	Platform string
-	File     *asset.File
+	File *asset.File
 }
 
 var _ asset.WritableAsset = (*TerraformVariables)(nil)
@@ -46,8 +45,6 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 	worker := &machine.Worker{}
 	parents.Get(installConfig, bootstrap, master, worker)
 
-	t.Platform = installConfig.Config.Platform.Name()
-
 	bootstrapIgn := string(bootstrap.Files()[0].Data)
 
 	masterFiles := master.Files()
@@ -77,3 +74,14 @@ func (t *TerraformVariables) Files() []*asset.File {
 	}
 	return []*asset.File{}
 }
+
+// Load reads the terraform.tfvars from disk.
+func (t *TerraformVariables) Load(f asset.FileFetcher) (found bool, err error) {
+	file := f.FetchByName(tfvarsFilename)
+	if file == nil {
+		return false, nil
+	}
+
+	t.File = file
+	return true, nil
+}

pkg/asset/filefetcher.go

@@ -0,0 +1,84 @@
+package asset
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+)
+
+// FileFetcher fetches the asset files from disk.
+type FileFetcher interface {
+	// FetchByName returns the file with the given name.
+	FetchByName(string) *File
+	// FetchByPattern returns the files whose name match the given regexp.
+	FetchByPattern(*regexp.Regexp) []*File
+}
+
+type fileFetcher struct {
+	onDiskAssets map[string][]byte
+}
+
+func newFileFetcher(clusterDir string) (*fileFetcher, error) {
+	fileMap := make(map[string][]byte)
+
+	// Don't bother if the clusterDir is not created yet because that
+	// means there's no assets generated yet.
+	_, err := os.Stat(clusterDir)
+	if err != nil && os.IsNotExist(err) {
+		return &fileFetcher{}, nil
+	}
+
+	if err := filepath.Walk(clusterDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if info.IsDir() {
+			return nil
+		}
+
+		filename, err := filepath.Rel(clusterDir, path)
+		if err != nil {
+			return err
+		}
+
+		data, err := ioutil.ReadFile(path)
+		if err != nil {
+			return err
+		}
+
+		fileMap[filename] = data
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+	return &fileFetcher{onDiskAssets: fileMap}, nil
+}
+
+// FetchByName returns the file with the given name.
+func (f *fileFetcher) FetchByName(name string) *File {
+	data, ok := f.onDiskAssets[name]
+	if !ok {
+		return nil
+	}
+	return &File{Filename: name, Data: data}
+}
+
+// FetchByPattern returns the files whose name match the given regexp.
+func (f *fileFetcher) FetchByPattern(re *regexp.Regexp) []*File {
+	var files []*File
+
+	for filename, data := range f.onDiskAssets {
+		if re.MatchString(filename) {
+			files = append(files, &File{
+				Filename: filename,
+				Data:     data,
+			})
+		}
+	}
+
+	sort.Slice(files, func(i, j int) bool { return files[i].Filename < files[j].Filename })
+	return files
+}