feat: use stacks schema merger (#1770)

* chore: make provider schema available to schema merger
* refactor: move PreloadEmbeddedSchema job into Defer as future jobs in Defer will depend on it
* feat: make module meta available in schema merger
* chore: add changie entry
* fix: combine readers into one, address review feedback
* refactor: continue parsing if loading stack component sources failed
* Bump terraform-schema to `9fc1ce5`

.changes/unreleased/ENHANCEMENTS-20240725-163756.yaml

@@ -0,0 +1,6 @@
+kind: ENHANCEMENTS
+body: Merge stack configuration schema with dynamic schema based on used components source and providers
+time: 2024-07-25T16:37:56.096322+02:00
+custom:
+    Issue: "1770"
+    Repository: terraform-ls

go.mod

@@ -13,12 +13,12 @@ require (
 	github.com/hashicorp/go-uuid v1.0.3
 	github.com/hashicorp/go-version v1.7.0
 	github.com/hashicorp/hc-install v0.7.0
-	github.com/hashicorp/hcl-lang v0.0.0-20240605150436-0e930f47b31b
+	github.com/hashicorp/hcl-lang v0.0.0-20240722075100-0f6cd5960306
 	github.com/hashicorp/hcl/v2 v2.21.0
 	github.com/hashicorp/terraform-exec v0.21.0
 	github.com/hashicorp/terraform-json v0.22.1
 	github.com/hashicorp/terraform-registry-address v0.2.3
-	github.com/hashicorp/terraform-schema v0.0.0-20240717154107-7d112f69e8e0
+	github.com/hashicorp/terraform-schema v0.0.0-20240730123044-9fc1ce56945b
 	github.com/mcuadros/go-defaults v1.2.0
 	github.com/mh-cbon/go-fmt-fail v0.0.0-20160815164508-67765b3fbcb5
 	github.com/mitchellh/cli v1.1.5
@@ -28,7 +28,7 @@ require (
 	github.com/pmezard/go-difflib v1.0.0
 	github.com/stretchr/testify v1.9.0
 	github.com/vektra/mockery/v2 v2.43.2
-	github.com/zclconf/go-cty v1.14.4
+	github.com/zclconf/go-cty v1.15.0
 	github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940
 	go.bobheadxi.dev/gobenchdata v1.3.1
 	go.opentelemetry.io/otel/trace v1.28.0

go.sum

@@ -219,8 +219,8 @@ github.com/hashicorp/hc-install v0.7.0 h1:Uu9edVqjKQxxuD28mR5TikkKDd/p55S8vzPC16
 github.com/hashicorp/hc-install v0.7.0/go.mod h1:ELmmzZlGnEcqoUMKUuykHaPCIR1sYLYX+KSggWSKZuA=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hashicorp/hcl-lang v0.0.0-20240605150436-0e930f47b31b h1:DRIJYSwLKoAfdndwIH1NbjLC3wm/RuyT7eEtm8aKw1U=
-github.com/hashicorp/hcl-lang v0.0.0-20240605150436-0e930f47b31b/go.mod h1:/g6sedjVJX99knsqTKU9wSWBVtsyDKWJkseNV9Zx1aU=
+github.com/hashicorp/hcl-lang v0.0.0-20240722075100-0f6cd5960306 h1:VQ2epS4r2HtNzKJhJVR8sIHdsMV7L80lCvwJkX02hd0=
+github.com/hashicorp/hcl-lang v0.0.0-20240722075100-0f6cd5960306/go.mod h1:4uIIEYVWTw+fDs9LVKy1CPnL6Z0XcLOjmanj7fVCIpE=
 github.com/hashicorp/hcl/v2 v2.21.0 h1:lve4q/o/2rqwYOgUg3y3V2YPyD1/zkCLGjIV74Jit14=
 github.com/hashicorp/hcl/v2 v2.21.0/go.mod h1:62ZYHrXgPoX8xBnzl8QzbWq4dyDsDtfCRgIq1rbJEvA=
 github.com/hashicorp/terraform-exec v0.21.0 h1:uNkLAe95ey5Uux6KJdua6+cv8asgILFVWkd/RG0D2XQ=
@@ -229,8 +229,8 @@ github.com/hashicorp/terraform-json v0.22.1 h1:xft84GZR0QzjPVWs4lRUwvTcPnegqlyS7
 github.com/hashicorp/terraform-json v0.22.1/go.mod h1:JbWSQCLFSXFFhg42T7l9iJwdGXBYV8fmmD6o/ML4p3A=
 github.com/hashicorp/terraform-registry-address v0.2.3 h1:2TAiKJ1A3MAkZlH1YI/aTVcLZRu7JseiXNRHbOAyoTI=
 github.com/hashicorp/terraform-registry-address v0.2.3/go.mod h1:lFHA76T8jfQteVfT7caREqguFrW3c4MFSPhZB7HHgUM=
-github.com/hashicorp/terraform-schema v0.0.0-20240717154107-7d112f69e8e0 h1:qgbO+ZzDng3AOKZ8GQ+BuoF24hYNM6EzaQMK8ICPbuw=
-github.com/hashicorp/terraform-schema v0.0.0-20240717154107-7d112f69e8e0/go.mod h1:ar787Bv/qD6tlnjtzH8fQ1Yi6c/B5LsnpFlO8c92Atg=
+github.com/hashicorp/terraform-schema v0.0.0-20240730123044-9fc1ce56945b h1:NlRWtn37AasCvxZH1OQjClxVEzG4C7qCNwi+2iXczVg=
+github.com/hashicorp/terraform-schema v0.0.0-20240730123044-9fc1ce56945b/go.mod h1:k86q3epNJLLo/iHmXVrdjLU84bprKTFW1vxuP4iG07M=
 github.com/hashicorp/terraform-svchost v0.1.1 h1:EZZimZ1GxdqFRinZ1tpJwVxxt49xc/S52uzrw4x0jKQ=
 github.com/hashicorp/terraform-svchost v0.1.1/go.mod h1:mNsjQfZyf/Jhz35v6/0LWcv26+X7JPS+buii2c9/ctc=
 github.com/hexops/autogold v1.3.1 h1:YgxF9OHWbEIUjhDbpnLhgVsjUDsiHDTyDfy2lrfdlzo=
@@ -384,8 +384,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-github.com/zclconf/go-cty v1.14.4 h1:uXXczd9QDGsgu0i/QFR/hzI5NYCHLf6NQw/atrbnhq8=
-github.com/zclconf/go-cty v1.14.4/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE=
+github.com/zclconf/go-cty v1.15.0 h1:tTCRWxsexYUmtt/wVxgDClUe+uQusuI443uL6e+5sXQ=
+github.com/zclconf/go-cty v1.15.0/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE=
 github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo=
 github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM=
 go.bobheadxi.dev/gobenchdata v1.3.1 h1:3Pts2nPUZdgFSU63nWzvfs2xRbK8WVSNeJ2H9e/Ypew=

internal/features/modules/modules_feature.go

@@ -283,3 +283,7 @@ func (f *ModulesFeature) MetadataReady(dir document.DirHandle) (<-chan struct{},
 
 	return f.Store.MetadataReady(dir)
 }
+
+func (s *ModulesFeature) LocalModuleMeta(modPath string) (*tfmod.Meta, error) {
+	return s.Store.LocalModuleMeta(modPath)
+}

internal/features/stacks/decoder/path_reader.go

@@ -7,23 +7,43 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/hashicorp/go-version"
 	"github.com/hashicorp/hcl-lang/decoder"
 	"github.com/hashicorp/hcl-lang/lang"
 	"github.com/hashicorp/hcl-lang/reference"
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/terraform-ls/internal/features/stacks/ast"
 	"github.com/hashicorp/terraform-ls/internal/features/stacks/state"
 	ilsp "github.com/hashicorp/terraform-ls/internal/lsp"
-	stackschema "github.com/hashicorp/terraform-schema/schema"
+	tfaddr "github.com/hashicorp/terraform-registry-address"
+	tfmod "github.com/hashicorp/terraform-schema/module"
+	tfschema "github.com/hashicorp/terraform-schema/schema"
+	stackschema "github.com/hashicorp/terraform-schema/schema/stacks"
+	tfstack "github.com/hashicorp/terraform-schema/stack"
 )
 
 type PathReader struct {
-	StateReader StateReader
+	StateReader  StateReader
+	ModuleReader ModuleReader
+}
+
+var _ decoder.PathReader = &PathReader{}
+
+type CombinedReader struct {
+	ModuleReader
+	StateReader
 }
 
 type StateReader interface {
 	List() ([]*state.StackRecord, error)
 	StackRecordByPath(modPath string) (*state.StackRecord, error)
+	ProviderSchema(modPath string, addr tfaddr.Provider, vc version.Constraints) (*tfschema.ProviderSchema, error)
+}
+
+type ModuleReader interface {
+	// LocalModuleMeta returns the module meta data for a local module. This is the result
+	// of the [earlydecoder] when processing module files
+	LocalModuleMeta(modPath string) (*tfmod.Meta, error)
 }
 
 // PathContext returns a PathContext for the given path based on the language ID
@@ -35,28 +55,48 @@ func (pr *PathReader) PathContext(path lang.Path) (*decoder.PathContext, error)
 
 	switch path.LanguageID {
 	case ilsp.Stacks.String():
-		return stackPathContext(record)
+		return stackPathContext(record, CombinedReader{
+			StateReader:  pr.StateReader,
+			ModuleReader: pr.ModuleReader,
+		})
 	case ilsp.Deploy.String():
 		return deployPathContext(record)
 	}
 
 	return nil, fmt.Errorf("unknown language ID: %q", path.LanguageID)
 }
 
-func stackPathContext(record *state.StackRecord) (*decoder.PathContext, error) {
+func stackPathContext(record *state.StackRecord, stateReader CombinedReader) (*decoder.PathContext, error) {
 	// TODO: this should only work for terraform 1.8 and above
 	version := record.RequiredTerraformVersion
 	if version == nil {
-		version = stackschema.LatestAvailableVersion
+		version = tfschema.LatestAvailableVersion
 	}
 
 	schema, err := stackschema.CoreStackSchemaForVersion(version)
 	if err != nil {
 		return nil, err
 	}
 
+	sm := stackschema.NewStackSchemaMerger(schema)
+	sm.SetStateReader(stateReader)
+
+	meta := &tfstack.Meta{
+		Path:                 record.Path(),
+		ProviderRequirements: record.Meta.ProviderRequirements,
+		Components:           record.Meta.Components,
+		Variables:            record.Meta.Variables,
+		Outputs:              record.Meta.Outputs,
+		Filenames:            record.Meta.Filenames,
+	}
+
+	mergedSchema, err := sm.SchemaForModule(meta)
+	if err != nil {
+		return nil, err
+	}
+
 	pathCtx := &decoder.PathContext{
-		Schema:           schema,
+		Schema:           mergedSchema,
 		ReferenceOrigins: make(reference.Origins, 0),
 		ReferenceTargets: make(reference.Targets, 0),
 		Files:            make(map[string]*hcl.File, 0),
@@ -77,7 +117,7 @@ func deployPathContext(record *state.StackRecord) (*decoder.PathContext, error)
 	// TODO: this should only work for terraform 1.8 and above
 	version := record.RequiredTerraformVersion
 	if version == nil {
-		version = stackschema.LatestAvailableVersion
+		version = tfschema.LatestAvailableVersion
 	}
 
 	schema, err := stackschema.CoreDeploySchemaForVersion(version)

internal/features/stacks/events.go

@@ -200,39 +200,46 @@ func (f *StacksFeature) decodeStack(ctx context.Context, dir document.DirHandle,
 		DependsOn:   job.IDs{parseId},
 		IgnoreState: ignoreState,
 		Defer: func(ctx context.Context, jobErr error) (job.IDs, error) {
+			deferIds := make(job.IDs, 0)
+
 			if jobErr != nil {
 				f.logger.Printf("loading module metadata returned error: %s", jobErr)
 			}
 
 			spawnedIds, err := loadStackComponentSources(ctx, f.store, f.bus, path)
+			deferIds = append(deferIds, spawnedIds...)
+			if err != nil {
+				f.logger.Printf("loading stack component sources returned error: %s", err)
+			}
 
 			// while we now have the job ids in here, depending on the metaId job is not enough
 			// to await these spawned jobs, so we will need to move all depending jobs to this function
 			// as well. e.g. LoadStackComponentSources, PreloadEmbeddedSchema (because future ref collection jobs depend on it), etc.
 			// we might just move all in here for simplicity
 
-			return spawnedIds, err
-		},
-	})
-	if err != nil {
-		return ids, err
-	}
-	ids = append(ids, metaId)
+			// Reference collection jobs will depend on this one, so we move it here in advance
+			eSchemaId, err := f.stateStore.JobStore.EnqueueJob(ctx, job.Job{
+				Dir: dir,
+				Func: func(ctx context.Context) error {
+					return jobs.PreloadEmbeddedSchema(ctx, f.logger, schemas.FS,
+						f.store, f.stateStore.ProviderSchemas, path)
+				},
+				// DependsOn: none required, since we are inside
+				Type:        operation.OpTypeStacksPreloadEmbeddedSchema.String(),
+				IgnoreState: ignoreState,
+			})
+			if err != nil {
+				return deferIds, err
+			}
+			deferIds = append(deferIds, eSchemaId)
 
-	eSchemaId, err := f.stateStore.JobStore.EnqueueJob(ctx, job.Job{
-		Dir: dir,
-		Func: func(ctx context.Context) error {
-			return jobs.PreloadEmbeddedSchema(ctx, f.logger, schemas.FS,
-				f.store, f.stateStore.ProviderSchemas, path)
+			return deferIds, nil
 		},
-		DependsOn:   job.IDs{metaId},
-		Type:        operation.OpTypeStacksPreloadEmbeddedSchema.String(),
-		IgnoreState: ignoreState,
 	})
 	if err != nil {
 		return ids, err
 	}
-	ids = append(ids, eSchemaId)
+	ids = append(ids, metaId)
 
 	// TODO: Implement the following functions where appropriate to stacks
 	// Future: decodeDeclaredModuleCalls(ctx, dir, ignoreState)

internal/features/stacks/stacks_feature.go

@@ -26,22 +26,25 @@ type StacksFeature struct {
 	fs         jobs.ReadOnlyFS
 	logger     *log.Logger
 	stopFunc   context.CancelFunc
+
+	moduleFeature stackDecoder.ModuleReader
 }
 
-func NewStacksFeature(bus *eventbus.EventBus, stateStore *globalState.StateStore, fs jobs.ReadOnlyFS) (*StacksFeature, error) {
+func NewStacksFeature(bus *eventbus.EventBus, stateStore *globalState.StateStore, fs jobs.ReadOnlyFS, moduleFeature stackDecoder.ModuleReader) (*StacksFeature, error) {
 	store, err := state.NewStackStore(stateStore.ChangeStore, stateStore.ProviderSchemas)
 	if err != nil {
 		return nil, err
 	}
 	discardLogger := log.New(io.Discard, "", 0)
 
 	return &StacksFeature{
-		store:      store,
-		bus:        bus,
-		fs:         fs,
-		stateStore: stateStore,
-		logger:     discardLogger,
-		stopFunc:   func() {},
+		store:         store,
+		bus:           bus,
+		fs:            fs,
+		stateStore:    stateStore,
+		logger:        discardLogger,
+		stopFunc:      func() {},
+		moduleFeature: moduleFeature,
 	}, nil
 }
 
@@ -100,15 +103,17 @@ func (f *StacksFeature) Stop() {
 
 func (f *StacksFeature) PathContext(path lang.Path) (*decoder.PathContext, error) {
 	pathReader := &stackDecoder.PathReader{
-		StateReader: f.store,
+		StateReader:  f.store,
+		ModuleReader: f.moduleFeature,
 	}
 
 	return pathReader.PathContext(path)
 }
 
 func (f *StacksFeature) Paths(ctx context.Context) []lang.Path {
 	pathReader := &stackDecoder.PathReader{
-		StateReader: f.store,
+		StateReader:  f.store,
+		ModuleReader: f.moduleFeature,
 	}
 
 	return pathReader.Paths(ctx)

internal/features/stacks/state/stack_store.go

@@ -13,6 +13,8 @@ import (
 	globalState "github.com/hashicorp/terraform-ls/internal/state"
 	globalAst "github.com/hashicorp/terraform-ls/internal/terraform/ast"
 	"github.com/hashicorp/terraform-ls/internal/terraform/module/operation"
+	tfaddr "github.com/hashicorp/terraform-registry-address"
+	tfschema "github.com/hashicorp/terraform-schema/schema"
 	tfstack "github.com/hashicorp/terraform-schema/stack"
 )
 
@@ -427,3 +429,7 @@ func (s *StackStore) queueRecordChange(oldRecord, newRecord *StackRecord) error
 
 	return s.changeStore.QueueChange(dir, changes)
 }
+
+func (s *StackStore) ProviderSchema(modPath string, addr tfaddr.Provider, vc version.Constraints) (*tfschema.ProviderSchema, error) {
+	return s.providerSchemasStore.ProviderSchema(modPath, addr, vc)
+}

internal/langserver/handlers/service.go

@@ -536,7 +536,7 @@ func (svc *service) configureSessionDependencies(ctx context.Context, cfgOpts *s
 		variablesFeature.SetLogger(svc.logger)
 		variablesFeature.Start(svc.sessCtx)
 
-		stacksFeature, err := stacks.NewStacksFeature(svc.eventBus, svc.stateStore, svc.fs)
+		stacksFeature, err := stacks.NewStacksFeature(svc.eventBus, svc.stateStore, svc.fs, modulesFeature)
 		if err != nil {
 			return err
 		}

internal/langserver/handlers/session_mock_test.go

@@ -162,7 +162,7 @@ func NewTestFeatures(eventBus *eventbus.EventBus, s *state.StateStore, fs *files
 		return nil, err
 	}
 
-	stacksFeature, err := fstacks.NewStacksFeature(eventBus, s, fs)
+	stacksFeature, err := fstacks.NewStacksFeature(eventBus, s, fs, modulesFeature)
 	if err != nil {
 		return nil, err
 	}