stacks: include existing components when deferring nested stacks

Author: liamcervante

internal/stacks/stackaddrs/in_stack.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/hclsyntax"
+
 	"github.com/hashicorp/terraform/internal/addrs"
 	"github.com/hashicorp/terraform/internal/collections"
 	"github.com/hashicorp/terraform/internal/tfdiags"
@@ -182,8 +183,8 @@ Steps:
 		}
 		traversal = traversal[2:] // consume the first two steps that we already dealt with
 		if len(traversal) > 0 {
-			idxStep, ok := traversal[0].(hcl.TraverseIndex)
-			if ok {
+			switch idxStep := traversal[0].(type) {
+			case hcl.TraverseIndex:
 				var err error
 				addrStep.Key, err = addrs.ParseInstanceKey(idxStep.Key)
 				if err != nil {
@@ -196,6 +197,9 @@ Steps:
 					return nil, nil, diags
 				}
 				traversal = traversal[1:] // consume the step we just dealt with
+			case hcl.TraverseSplat:
+				addrStep.Key = addrs.WildcardKey
+				traversal = traversal[1:]
 			}
 		}
 		stackInst = append(stackInst, addrStep)

internal/stacks/stackplan/plan.go

@@ -107,6 +107,27 @@ func (p *Plan) ComponentInstances(addr stackaddrs.AbsComponent) collections.Set[
 	return ret
 }
 
+func (p *Plan) StackInstances(addr stackaddrs.AbsStackCall) map[stackaddrs.StackInstanceStep]bool {
+	ret := make(map[stackaddrs.StackInstanceStep]bool)
+	for key := range p.Components.All() {
+		if len(key.Stack) == 0 {
+			continue
+		}
+
+		last := key.Stack[len(key.Stack)-1]
+		path := key.Stack[:len(key.Stack)-1]
+
+		if path.String() != addr.Stack.String() {
+			continue
+		}
+		if last.Name != addr.Item.Name {
+			continue
+		}
+		ret[last] = true
+	}
+	return ret
+}
+
 // RequiredProviderInstances returns a description of all of the provider
 // instance slots that are required to satisfy the resource instances
 // belonging to the given component instance.

internal/stacks/stackruntime/internal/stackeval/component.go

@@ -144,7 +144,7 @@ func (c *Component) CheckInstances(ctx context.Context, phase EvalPhase) (map[ad
 			}
 
 			result := instancesMap(forEachVal, func(ik addrs.InstanceKey, rd instances.RepetitionData) *ComponentInstance {
-				return newComponentInstance(c, stackaddrs.AbsComponentToInstance(c.addr, ik), rd, false)
+				return newComponentInstance(c, stackaddrs.AbsComponentToInstance(c.addr, ik), rd, c.stack.deferred)
 			})
 
 			addrs := make([]stackaddrs.AbsComponentInstance, 0, len(result.insts))

internal/stacks/stackruntime/internal/stackeval/main.go

@@ -310,7 +310,7 @@ func (m *Main) MainStack() *Stack {
 	defer m.mu.Unlock()
 
 	if m.mainStack == nil {
-		m.mainStack = newStack(m, stackaddrs.RootStackInstance, nil, config, collections.NewMap[stackaddrs.ConfigComponent, []*RemovedComponent]())
+		m.mainStack = newStack(m, stackaddrs.RootStackInstance, nil, config, collections.NewMap[stackaddrs.ConfigComponent, []*RemovedComponent](), false)
 	}
 	return m.mainStack
 }

internal/stacks/stackruntime/internal/stackeval/removed_component.go

@@ -111,7 +111,7 @@ func (r *RemovedComponent) Instances(ctx context.Context, phase EvalPhase) (map[
 				return nil
 			}
 
-			return newRemovedComponentInstance(r, addr, rd, false)
+			return newRemovedComponentInstance(r, addr, rd, r.stack.deferred)
 		})
 
 		// Now, filter out any instances that are not known to the previous

internal/stacks/stackruntime/internal/stackeval/stack.go

@@ -29,10 +29,11 @@ import (
 // repetition arguments (if any) evaluated to determine how many instances
 // it has.
 type Stack struct {
-	addr   stackaddrs.StackInstance
-	parent *Stack
-	config *StackConfig
-	main   *Main
+	addr     stackaddrs.StackInstance
+	parent   *Stack
+	config   *StackConfig
+	main     *Main
+	deferred bool
 
 	// externalRemovedComponents are the set of removed blocks that might
 	// target components within either this stack or a child of this stack.
@@ -56,11 +57,12 @@ var (
 	_ Applyable       = (*Stack)(nil)
 )
 
-func newStack(main *Main, addr stackaddrs.StackInstance, parent *Stack, config *StackConfig, removedComponents collections.Map[stackaddrs.ConfigComponent, []*RemovedComponent]) *Stack {
+func newStack(main *Main, addr stackaddrs.StackInstance, parent *Stack, config *StackConfig, removedComponents collections.Map[stackaddrs.ConfigComponent, []*RemovedComponent], deferred bool) *Stack {
 	return &Stack{
 		parent:                    parent,
 		config:                    config,
 		addr:                      addr,
+		deferred:                  deferred,
 		main:                      main,
 		externalRemovedComponents: removedComponents,
 	}
@@ -201,6 +203,25 @@ func (s *Stack) EmbeddedStackCall(addr stackaddrs.StackCall) *StackCall {
 	return s.EmbeddedStackCalls()[addr]
 }
 
+func (s *Stack) KnownEmbeddedStacks(addr stackaddrs.StackCall, phase EvalPhase) map[stackaddrs.StackInstanceStep]bool {
+	switch phase {
+	case PlanPhase:
+		return s.main.PlanPrevState().StackInstances(stackaddrs.AbsStackCall{
+			Stack: s.addr,
+			Item:  addr,
+		})
+	case ApplyPhase:
+		return s.main.PlanBeingApplied().StackInstances(stackaddrs.AbsStackCall{
+			Stack: s.addr,
+			Item:  addr,
+		})
+	default:
+		// We're not executing with an existing state in the other phases, so
+		// we have no known instances.
+		return nil
+	}
+}
+
 func (s *Stack) Components() map[stackaddrs.Component]*Component {
 	s.mu.Lock()
 	defer s.mu.Unlock()

internal/stacks/stackruntime/internal/stackeval/stack_call.go

@@ -150,7 +150,7 @@ func (c *StackCall) CheckInstances(ctx context.Context, phase EvalPhase) (map[ad
 			}
 
 			return instancesMap(forEachVal, func(ik addrs.InstanceKey, rd instances.RepetitionData) *StackCallInstance {
-				return newStackCallInstance(c, ik, rd)
+				return newStackCallInstance(c, ik, rd, c.stack.deferred)
 			}), diags
 		},
 	)
@@ -159,7 +159,7 @@ func (c *StackCall) CheckInstances(ctx context.Context, phase EvalPhase) (map[ad
 
 func (c *StackCall) UnknownInstance(ctx context.Context, phase EvalPhase) *StackCallInstance {
 	inst, err := c.unknownInstance.For(phase).Do(ctx, c.tracingName()+" unknown instace", func(ctx context.Context) (*StackCallInstance, error) {
-		return newStackCallInstance(c, addrs.WildcardKey, instances.UnknownForEachRepetitionData(c.ForEachValue(ctx, phase).Type())), nil
+		return newStackCallInstance(c, addrs.WildcardKey, instances.UnknownForEachRepetitionData(c.ForEachValue(ctx, phase).Type()), true), nil
 	})
 	if err != nil {
 		// Since we never return an error from the function we pass to Do,

internal/stacks/stackruntime/internal/stackeval/stack_call_instance.go

@@ -31,8 +31,9 @@ import (
 // are holding a valid [StackCallInstance] then you can call
 // [StackCallInstance.CalledStack] to get that stack.
 type StackCallInstance struct {
-	call *StackCall
-	key  addrs.InstanceKey
+	call     *StackCall
+	key      addrs.InstanceKey
+	deferred bool
 
 	main *Main
 
@@ -45,10 +46,11 @@ type StackCallInstance struct {
 var _ ExpressionScope = (*StackCallInstance)(nil)
 var _ Plannable = (*StackCallInstance)(nil)
 
-func newStackCallInstance(call *StackCall, key addrs.InstanceKey, repetition instances.RepetitionData) *StackCallInstance {
+func newStackCallInstance(call *StackCall, key addrs.InstanceKey, repetition instances.RepetitionData, deferred bool) *StackCallInstance {
 	return &StackCallInstance{
 		call:       call,
 		key:        key,
+		deferred:   deferred,
 		main:       call.main,
 		repetition: repetition,
 	}
@@ -67,7 +69,7 @@ func (c *StackCallInstance) CalledStackAddr() stackaddrs.StackInstance {
 
 func (c *StackCallInstance) Stack(ctx context.Context, phase EvalPhase) *Stack {
 	stack, err := c.stack.For(phase).Do(ctx, c.tracingName(), func(ctx context.Context) (*Stack, error) {
-		return newStack(c.main, c.CalledStackAddr(), c.call.stack, c.call.config.TargetConfig(), c.call.GetExternalRemovedBlocks()), nil
+		return newStack(c.main, c.CalledStackAddr(), c.call.stack, c.call.config.TargetConfig(), c.call.GetExternalRemovedBlocks(), c.deferred), nil
 	})
 	if err != nil {
 		// we don't have cycles in here, and we don't return an error so this

internal/stacks/stackruntime/internal/stackeval/walk_dynamic.go

@@ -70,12 +70,39 @@ func walkDynamicObjectsInStack[Output any](
 
 			// If unknown, then process the unknown instance and skip the rest.
 			if unknown {
-				inst := call.UnknownInstance(ctx, phase)
-				visit(ctx, walk, inst)
+				knownInstances := stack.KnownEmbeddedStacks(call.addr.Item, phase)
+				if len(knownInstances) == 0 {
+					// with no known instances, we'll just process the constant
+					// unknown instance
+					inst := call.UnknownInstance(ctx, phase)
+					visit(ctx, walk, inst)
 
-				childStack := inst.Stack(ctx, phase)
-				walkDynamicObjectsInStack(ctx, walk, childStack, phase, visit)
-				return
+					childStack := inst.Stack(ctx, phase)
+					walkDynamicObjectsInStack(ctx, walk, childStack, phase, visit)
+					return
+				}
+
+				// otherwise we have instances, so we'll process those with
+				// dedicated instances
+
+				for inst := range knownInstances {
+					if inst.Key == addrs.WildcardKey {
+						inst := call.UnknownInstance(ctx, phase)
+						visit(ctx, walk, inst)
+
+						childStack := inst.Stack(ctx, phase)
+						walkDynamicObjectsInStack(ctx, walk, childStack, phase, visit)
+					} else {
+						inst := newStackCallInstance(call, inst.Key, instances.RepetitionData{
+							EachKey:   inst.Key.Value(),
+							EachValue: cty.UnknownVal(cty.DynamicPseudoType),
+						}, true)
+						visit(ctx, walk, inst)
+
+						childStack := inst.Stack(ctx, phase)
+						walkDynamicObjectsInStack(ctx, walk, childStack, phase, visit)
+					}
+				}
 			}
 
 			// Otherwise, process the instances and their child stacks.