terraform: new apply graph builder based on the "diff"

.travis.yml

@@ -10,6 +10,7 @@ install:
 - bash scripts/gogetcookie.sh
 script:
 - make test vet
+- make test TEST=./terraform TESTARGS=-Xnew-apply
 branches:
   only:
   - master

command/apply.go

@@ -95,6 +95,26 @@ func (c *ApplyCommand) Run(args []string) int {
 		}
 	}
 
+	// Check for the new apply
+	if terraform.X_newApply {
+		desc := "Experimental new apply graph has been enabled. This may still\n" +
+			"have bugs, and should be used with care. If you'd like to continue,\n" +
+			"you must enter exactly 'yes' as a response."
+		v, err := c.UIInput().Input(&terraform.InputOpts{
+			Id:          "Xnew-apply",
+			Query:       "Experimental feature enabled: new apply graph. Continue?",
+			Description: desc,
+		})
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("Error asking for confirmation: %s", err))
+			return 1
+		}
+		if v != "yes" {
+			c.Ui.Output("Apply cancelled.")
+			return 1
+		}
+	}
+
 	// Build the context based on the arguments given
 	ctx, planned, err := c.Context(contextOpts{
 		Destroy:     c.Destroy,

command/meta.go

@@ -328,6 +328,9 @@ func (m *Meta) flagSet(n string) *flag.FlagSet {
 		f.Var((*FlagKVFile)(&m.autoVariables), m.autoKey, "variable file")
 	}
 
+	// Experimental features
+	f.BoolVar(&terraform.X_newApply, "Xnew-apply", false, "experiment: new apply")
+
 	// Create an io.Writer that writes to our Ui properly for errors.
 	// This is kind of a hack, but it does the job. Basically: create
 	// a pipe, use a scanner to break it into lines, and output each line

config/module/tree.go

@@ -66,6 +66,10 @@ func (t *Tree) Config() *config.Config {
 
 // Child returns the child with the given path (by name).
 func (t *Tree) Child(path []string) *Tree {
+	if t == nil {
+		return nil
+	}
+
 	if len(path) == 0 {
 		return t
 	}

config/module/tree_test.go

@@ -12,6 +12,11 @@ import (
 )
 
 func TestTreeChild(t *testing.T) {
+	var nilTree *Tree
+	if nilTree.Child(nil) != nil {
+		t.Fatal("child should be nil")
+	}
+
 	storage := testStorage(t)
 	tree := NewTree("", testConfig(t, "child"))
 	if err := tree.Load(storage, GetModeGet); err != nil {

config/raw_config.go

@@ -191,17 +191,19 @@ func (r *RawConfig) Merge(other *RawConfig) *RawConfig {
 	}
 
 	// Build the unknown keys
-	unknownKeys := make(map[string]struct{})
-	for _, k := range r.unknownKeys {
-		unknownKeys[k] = struct{}{}
-	}
-	for _, k := range other.unknownKeys {
-		unknownKeys[k] = struct{}{}
-	}
+	if len(r.unknownKeys) > 0 || len(other.unknownKeys) > 0 {
+		unknownKeys := make(map[string]struct{})
+		for _, k := range r.unknownKeys {
+			unknownKeys[k] = struct{}{}
+		}
+		for _, k := range other.unknownKeys {
+			unknownKeys[k] = struct{}{}
+		}
 
-	result.unknownKeys = make([]string, 0, len(unknownKeys))
-	for k, _ := range unknownKeys {
-		result.unknownKeys = append(result.unknownKeys, k)
+		result.unknownKeys = make([]string, 0, len(unknownKeys))
+		for k, _ := range unknownKeys {
+			result.unknownKeys = append(result.unknownKeys, k)
+		}
 	}
 
 	return result

config/raw_config_test.go

@@ -27,7 +27,7 @@ func TestNewRawConfig(t *testing.T) {
 	}
 }
 
-func TestRawConfig(t *testing.T) {
+func TestRawConfig_basic(t *testing.T) {
 	raw := map[string]interface{}{
 		"foo": "${var.bar}",
 	}

config/testing.go

@@ -0,0 +1,15 @@
+package config
+
+import (
+	"testing"
+)
+
+// TestRawConfig is used to create a RawConfig for testing.
+func TestRawConfig(t *testing.T, c map[string]interface{}) *RawConfig {
+	cfg, err := NewRawConfig(c)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	return cfg
+}

dag/graph.go

@@ -48,6 +48,32 @@ func (g *Graph) Edges() []Edge {
 	return result
 }
 
+// EdgesFrom returns the list of edges from the given source.
+func (g *Graph) EdgesFrom(v Vertex) []Edge {
+	var result []Edge
+	from := hashcode(v)
+	for _, e := range g.Edges() {
+		if hashcode(e.Source()) == from {
+			result = append(result, e)
+		}
+	}
+
+	return result
+}
+
+// EdgesTo returns the list of edges to the given target.
+func (g *Graph) EdgesTo(v Vertex) []Edge {
+	var result []Edge
+	search := hashcode(v)
+	for _, e := range g.Edges() {
+		if hashcode(e.Target()) == search {
+			result = append(result, e)
+		}
+	}
+
+	return result
+}
+
 // HasVertex checks if the given Vertex is present in the graph.
 func (g *Graph) HasVertex(v Vertex) bool {
 	return g.vertices.Include(v)

dag/graph_test.go

@@ -124,6 +124,52 @@ func TestGraphHasEdge(t *testing.T) {
 	}
 }
 
+func TestGraphEdgesFrom(t *testing.T) {
+	var g Graph
+	g.Add(1)
+	g.Add(2)
+	g.Add(3)
+	g.Connect(BasicEdge(1, 3))
+	g.Connect(BasicEdge(2, 3))
+
+	edges := g.EdgesFrom(1)
+
+	var expected Set
+	expected.Add(BasicEdge(1, 3))
+
+	var s Set
+	for _, e := range edges {
+		s.Add(e)
+	}
+
+	if s.Intersection(&expected).Len() != expected.Len() {
+		t.Fatalf("bad: %#v", edges)
+	}
+}
+
+func TestGraphEdgesTo(t *testing.T) {
+	var g Graph
+	g.Add(1)
+	g.Add(2)
+	g.Add(3)
+	g.Connect(BasicEdge(1, 3))
+	g.Connect(BasicEdge(1, 2))
+
+	edges := g.EdgesTo(3)
+
+	var expected Set
+	expected.Add(BasicEdge(1, 3))
+
+	var s Set
+	for _, e := range edges {
+		s.Add(e)
+	}
+
+	if s.Intersection(&expected).Len() != expected.Len() {
+		t.Fatalf("bad: %#v", edges)
+	}
+}
+
 type hashVertex struct {
 	code interface{}
 }

go.sh

@@ -0,0 +1 @@
+go test ./terraform | grep -E '(FAIL|panic)' | tee /dev/tty | wc -l

terraform/context.go

@@ -13,6 +13,16 @@ import (
 	"github.com/hashicorp/terraform/config/module"
 )
 
+// Variables prefixed with X_ are experimental features. They can be enabled
+// by setting them to true. This should be done before any API is called.
+// These should be expected to be removed at some point in the future; each
+// option should mention a schedule.
+var (
+	// X_newApply will enable the new apply graph. This will be removed
+	// and be on by default in 0.8.0.
+	X_newApply = false
+)
+
 // InputMode defines what sort of input will be asked for when Input
 // is called on Context.
 type InputMode byte
@@ -353,21 +363,79 @@ func (c *Context) Apply() (*State, error) {
 	// Copy our own state
 	c.state = c.state.DeepCopy()
 
-	// Build the graph
-	graph, err := c.Graph(&ContextGraphOpts{Validate: true})
+	// Build the original graph. This is before the new graph builders
+	// coming in 0.8. We do this for shadow graphing.
+	oldGraph, err := c.Graph(&ContextGraphOpts{Validate: true})
+	if err != nil && X_newApply {
+		// If we had an error graphing but we're using the new graph,
+		// just set it to nil and let it go. There are some features that
+		// may work with the new graph that don't with the old.
+		oldGraph = nil
+		err = nil
+	}
 	if err != nil {
 		return nil, err
 	}
 
-	// Do the walk
-	var walker *ContextGraphWalker
+	// Build the new graph. We do this no matter what so we can shadow it.
+	newGraph, err := (&ApplyGraphBuilder{
+		Module:       c.module,
+		Diff:         c.diff,
+		State:        c.state,
+		Providers:    c.components.ResourceProviders(),
+		Provisioners: c.components.ResourceProvisioners(),
+	}).Build(RootModulePath)
+	if err != nil && !X_newApply {
+		// If we had an error graphing but we're not using this graph, just
+		// set it to nil and record it as a shadow error.
+		c.shadowErr = multierror.Append(c.shadowErr, fmt.Errorf(
+			"Error building new apply graph: %s", err))
+
+		newGraph = nil
+		err = nil
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	// Determine what is the real and what is the shadow. The logic here
+	// is straightforward though the if statements are not:
+	//
+	//   * Destroy mode - always use original, shadow with nothing because
+	//     we're only testing the new APPLY graph.
+	//   * Apply with new apply - use new graph, shadow is new graph. We can't
+	//     shadow with the old graph because the old graph does a lot more
+	//     that it shouldn't.
+	//   * Apply with old apply - use old graph, shadow with new graph.
+	//
+	real := oldGraph
+	shadow := newGraph
 	if c.destroy {
-		walker, err = c.walk(graph, graph, walkDestroy)
+		log.Printf("[WARN] terraform: real graph is original, shadow is nil")
+		shadow = nil
 	} else {
-		//walker, err = c.walk(graph, nil, walkApply)
-		walker, err = c.walk(graph, graph, walkApply)
+		if X_newApply {
+			log.Printf("[WARN] terraform: real graph is Xnew-apply, shadow is Xnew-apply")
+			real = shadow
+		} else {
+			log.Printf("[WARN] terraform: real graph is original, shadow is Xnew-apply")
+		}
+	}
+
+	// Determine the operation
+	operation := walkApply
+	if c.destroy {
+		operation = walkDestroy
+	}
+
+	// This shouldn't happen, so assert it. This is before any state changes
+	// so it is safe to crash here.
+	if real == nil {
+		panic("nil real graph")
 	}
 
+	// Walk the graph
+	walker, err := c.walk(real, shadow, operation)
 	if len(walker.ValidationErrors) > 0 {
 		err = multierror.Append(err, walker.ValidationErrors...)
 	}