From 1ca689b3dd1e1da00a8c0f40ce20042782a0f003 Mon Sep 17 00:00:00 2001 From: Ansgar Mertens Date: Tue, 30 Jul 2024 14:43:23 +0200 Subject: [PATCH] feat: use stacks schema merger (#1770) * chore: make provider schema available to schema merger * refactor: move PreloadEmbeddedSchema job into Defer as future jobs in Defer will depend on it * feat: make module meta available in schema merger * chore: add changie entry * fix: combine readers into one, address review feedback * refactor: continue parsing if loading stack component sources failed * Bump terraform-schema to `9fc1ce5` --- .../ENHANCEMENTS-20240725-163756.yaml | 6 +++ go.mod | 6 +-- go.sum | 12 ++--- internal/features/modules/modules_feature.go | 4 ++ .../features/stacks/decoder/path_reader.go | 54 ++++++++++++++++--- internal/features/stacks/events.go | 39 ++++++++------ internal/features/stacks/stacks_feature.go | 23 ++++---- internal/features/stacks/state/stack_store.go | 6 +++ internal/langserver/handlers/service.go | 2 +- .../langserver/handlers/session_mock_test.go | 2 +- 10 files changed, 111 insertions(+), 43 deletions(-) create mode 100644 .changes/unreleased/ENHANCEMENTS-20240725-163756.yaml diff --git a/.changes/unreleased/ENHANCEMENTS-20240725-163756.yaml b/.changes/unreleased/ENHANCEMENTS-20240725-163756.yaml new file mode 100644 index 000000000..1c5ed5d00 --- /dev/null +++ b/.changes/unreleased/ENHANCEMENTS-20240725-163756.yaml @@ -0,0 +1,6 @@ +kind: ENHANCEMENTS +body: Merge stack configuration schema with dynamic schema based on used components source and providers +time: 2024-07-25T16:37:56.096322+02:00 +custom: + Issue: "1770" + Repository: terraform-ls diff --git a/go.mod b/go.mod index 989b66128..e4fc3f4d6 100644 --- a/go.mod +++ b/go.mod @@ -13,12 +13,12 @@ require ( github.com/hashicorp/go-uuid v1.0.3 github.com/hashicorp/go-version v1.7.0 github.com/hashicorp/hc-install v0.7.0 - github.com/hashicorp/hcl-lang v0.0.0-20240605150436-0e930f47b31b + github.com/hashicorp/hcl-lang v0.0.0-20240722075100-0f6cd5960306 github.com/hashicorp/hcl/v2 v2.21.0 github.com/hashicorp/terraform-exec v0.21.0 github.com/hashicorp/terraform-json v0.22.1 github.com/hashicorp/terraform-registry-address v0.2.3 - github.com/hashicorp/terraform-schema v0.0.0-20240717154107-7d112f69e8e0 + github.com/hashicorp/terraform-schema v0.0.0-20240730123044-9fc1ce56945b github.com/mcuadros/go-defaults v1.2.0 github.com/mh-cbon/go-fmt-fail v0.0.0-20160815164508-67765b3fbcb5 github.com/mitchellh/cli v1.1.5 @@ -28,7 +28,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 github.com/stretchr/testify v1.9.0 github.com/vektra/mockery/v2 v2.43.2 - github.com/zclconf/go-cty v1.14.4 + github.com/zclconf/go-cty v1.15.0 github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 go.bobheadxi.dev/gobenchdata v1.3.1 go.opentelemetry.io/otel/trace v1.28.0 diff --git a/go.sum b/go.sum index 3dacc9b5e..6815639b6 100644 --- a/go.sum +++ b/go.sum @@ -219,8 +219,8 @@ github.com/hashicorp/hc-install v0.7.0 h1:Uu9edVqjKQxxuD28mR5TikkKDd/p55S8vzPC16 github.com/hashicorp/hc-install v0.7.0/go.mod h1:ELmmzZlGnEcqoUMKUuykHaPCIR1sYLYX+KSggWSKZuA= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/hcl-lang v0.0.0-20240605150436-0e930f47b31b h1:DRIJYSwLKoAfdndwIH1NbjLC3wm/RuyT7eEtm8aKw1U= -github.com/hashicorp/hcl-lang v0.0.0-20240605150436-0e930f47b31b/go.mod h1:/g6sedjVJX99knsqTKU9wSWBVtsyDKWJkseNV9Zx1aU= +github.com/hashicorp/hcl-lang v0.0.0-20240722075100-0f6cd5960306 h1:VQ2epS4r2HtNzKJhJVR8sIHdsMV7L80lCvwJkX02hd0= +github.com/hashicorp/hcl-lang v0.0.0-20240722075100-0f6cd5960306/go.mod h1:4uIIEYVWTw+fDs9LVKy1CPnL6Z0XcLOjmanj7fVCIpE= github.com/hashicorp/hcl/v2 v2.21.0 h1:lve4q/o/2rqwYOgUg3y3V2YPyD1/zkCLGjIV74Jit14= github.com/hashicorp/hcl/v2 v2.21.0/go.mod h1:62ZYHrXgPoX8xBnzl8QzbWq4dyDsDtfCRgIq1rbJEvA= github.com/hashicorp/terraform-exec v0.21.0 h1:uNkLAe95ey5Uux6KJdua6+cv8asgILFVWkd/RG0D2XQ= @@ -229,8 +229,8 @@ github.com/hashicorp/terraform-json v0.22.1 h1:xft84GZR0QzjPVWs4lRUwvTcPnegqlyS7 github.com/hashicorp/terraform-json v0.22.1/go.mod h1:JbWSQCLFSXFFhg42T7l9iJwdGXBYV8fmmD6o/ML4p3A= github.com/hashicorp/terraform-registry-address v0.2.3 h1:2TAiKJ1A3MAkZlH1YI/aTVcLZRu7JseiXNRHbOAyoTI= github.com/hashicorp/terraform-registry-address v0.2.3/go.mod h1:lFHA76T8jfQteVfT7caREqguFrW3c4MFSPhZB7HHgUM= -github.com/hashicorp/terraform-schema v0.0.0-20240717154107-7d112f69e8e0 h1:qgbO+ZzDng3AOKZ8GQ+BuoF24hYNM6EzaQMK8ICPbuw= -github.com/hashicorp/terraform-schema v0.0.0-20240717154107-7d112f69e8e0/go.mod h1:ar787Bv/qD6tlnjtzH8fQ1Yi6c/B5LsnpFlO8c92Atg= +github.com/hashicorp/terraform-schema v0.0.0-20240730123044-9fc1ce56945b h1:NlRWtn37AasCvxZH1OQjClxVEzG4C7qCNwi+2iXczVg= +github.com/hashicorp/terraform-schema v0.0.0-20240730123044-9fc1ce56945b/go.mod h1:k86q3epNJLLo/iHmXVrdjLU84bprKTFW1vxuP4iG07M= github.com/hashicorp/terraform-svchost v0.1.1 h1:EZZimZ1GxdqFRinZ1tpJwVxxt49xc/S52uzrw4x0jKQ= github.com/hashicorp/terraform-svchost v0.1.1/go.mod h1:mNsjQfZyf/Jhz35v6/0LWcv26+X7JPS+buii2c9/ctc= github.com/hexops/autogold v1.3.1 h1:YgxF9OHWbEIUjhDbpnLhgVsjUDsiHDTyDfy2lrfdlzo= @@ -384,8 +384,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/zclconf/go-cty v1.14.4 h1:uXXczd9QDGsgu0i/QFR/hzI5NYCHLf6NQw/atrbnhq8= -github.com/zclconf/go-cty v1.14.4/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= +github.com/zclconf/go-cty v1.15.0 h1:tTCRWxsexYUmtt/wVxgDClUe+uQusuI443uL6e+5sXQ= +github.com/zclconf/go-cty v1.15.0/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo= github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM= go.bobheadxi.dev/gobenchdata v1.3.1 h1:3Pts2nPUZdgFSU63nWzvfs2xRbK8WVSNeJ2H9e/Ypew= diff --git a/internal/features/modules/modules_feature.go b/internal/features/modules/modules_feature.go index f555f8710..b86e925c2 100644 --- a/internal/features/modules/modules_feature.go +++ b/internal/features/modules/modules_feature.go @@ -283,3 +283,7 @@ func (f *ModulesFeature) MetadataReady(dir document.DirHandle) (<-chan struct{}, return f.Store.MetadataReady(dir) } + +func (s *ModulesFeature) LocalModuleMeta(modPath string) (*tfmod.Meta, error) { + return s.Store.LocalModuleMeta(modPath) +} diff --git a/internal/features/stacks/decoder/path_reader.go b/internal/features/stacks/decoder/path_reader.go index 07cdfcd22..4c0b72820 100644 --- a/internal/features/stacks/decoder/path_reader.go +++ b/internal/features/stacks/decoder/path_reader.go @@ -7,6 +7,7 @@ import ( "context" "fmt" + "github.com/hashicorp/go-version" "github.com/hashicorp/hcl-lang/decoder" "github.com/hashicorp/hcl-lang/lang" "github.com/hashicorp/hcl-lang/reference" @@ -14,16 +15,35 @@ import ( "github.com/hashicorp/terraform-ls/internal/features/stacks/ast" "github.com/hashicorp/terraform-ls/internal/features/stacks/state" ilsp "github.com/hashicorp/terraform-ls/internal/lsp" - stackschema "github.com/hashicorp/terraform-schema/schema" + tfaddr "github.com/hashicorp/terraform-registry-address" + tfmod "github.com/hashicorp/terraform-schema/module" + tfschema "github.com/hashicorp/terraform-schema/schema" + stackschema "github.com/hashicorp/terraform-schema/schema/stacks" + tfstack "github.com/hashicorp/terraform-schema/stack" ) type PathReader struct { - StateReader StateReader + StateReader StateReader + ModuleReader ModuleReader +} + +var _ decoder.PathReader = &PathReader{} + +type CombinedReader struct { + ModuleReader + StateReader } type StateReader interface { List() ([]*state.StackRecord, error) StackRecordByPath(modPath string) (*state.StackRecord, error) + ProviderSchema(modPath string, addr tfaddr.Provider, vc version.Constraints) (*tfschema.ProviderSchema, error) +} + +type ModuleReader interface { + // LocalModuleMeta returns the module meta data for a local module. This is the result + // of the [earlydecoder] when processing module files + LocalModuleMeta(modPath string) (*tfmod.Meta, error) } // PathContext returns a PathContext for the given path based on the language ID @@ -35,7 +55,10 @@ func (pr *PathReader) PathContext(path lang.Path) (*decoder.PathContext, error) switch path.LanguageID { case ilsp.Stacks.String(): - return stackPathContext(record) + return stackPathContext(record, CombinedReader{ + StateReader: pr.StateReader, + ModuleReader: pr.ModuleReader, + }) case ilsp.Deploy.String(): return deployPathContext(record) } @@ -43,11 +66,11 @@ func (pr *PathReader) PathContext(path lang.Path) (*decoder.PathContext, error) return nil, fmt.Errorf("unknown language ID: %q", path.LanguageID) } -func stackPathContext(record *state.StackRecord) (*decoder.PathContext, error) { +func stackPathContext(record *state.StackRecord, stateReader CombinedReader) (*decoder.PathContext, error) { // TODO: this should only work for terraform 1.8 and above version := record.RequiredTerraformVersion if version == nil { - version = stackschema.LatestAvailableVersion + version = tfschema.LatestAvailableVersion } schema, err := stackschema.CoreStackSchemaForVersion(version) @@ -55,8 +78,25 @@ func stackPathContext(record *state.StackRecord) (*decoder.PathContext, error) { return nil, err } + sm := stackschema.NewStackSchemaMerger(schema) + sm.SetStateReader(stateReader) + + meta := &tfstack.Meta{ + Path: record.Path(), + ProviderRequirements: record.Meta.ProviderRequirements, + Components: record.Meta.Components, + Variables: record.Meta.Variables, + Outputs: record.Meta.Outputs, + Filenames: record.Meta.Filenames, + } + + mergedSchema, err := sm.SchemaForModule(meta) + if err != nil { + return nil, err + } + pathCtx := &decoder.PathContext{ - Schema: schema, + Schema: mergedSchema, ReferenceOrigins: make(reference.Origins, 0), ReferenceTargets: make(reference.Targets, 0), Files: make(map[string]*hcl.File, 0), @@ -77,7 +117,7 @@ func deployPathContext(record *state.StackRecord) (*decoder.PathContext, error) // TODO: this should only work for terraform 1.8 and above version := record.RequiredTerraformVersion if version == nil { - version = stackschema.LatestAvailableVersion + version = tfschema.LatestAvailableVersion } schema, err := stackschema.CoreDeploySchemaForVersion(version) diff --git a/internal/features/stacks/events.go b/internal/features/stacks/events.go index 8b4ef6987..b7c955652 100644 --- a/internal/features/stacks/events.go +++ b/internal/features/stacks/events.go @@ -200,39 +200,46 @@ func (f *StacksFeature) decodeStack(ctx context.Context, dir document.DirHandle, DependsOn: job.IDs{parseId}, IgnoreState: ignoreState, Defer: func(ctx context.Context, jobErr error) (job.IDs, error) { + deferIds := make(job.IDs, 0) + if jobErr != nil { f.logger.Printf("loading module metadata returned error: %s", jobErr) } spawnedIds, err := loadStackComponentSources(ctx, f.store, f.bus, path) + deferIds = append(deferIds, spawnedIds...) + if err != nil { + f.logger.Printf("loading stack component sources returned error: %s", err) + } // while we now have the job ids in here, depending on the metaId job is not enough // to await these spawned jobs, so we will need to move all depending jobs to this function // as well. e.g. LoadStackComponentSources, PreloadEmbeddedSchema (because future ref collection jobs depend on it), etc. // we might just move all in here for simplicity - return spawnedIds, err - }, - }) - if err != nil { - return ids, err - } - ids = append(ids, metaId) + // Reference collection jobs will depend on this one, so we move it here in advance + eSchemaId, err := f.stateStore.JobStore.EnqueueJob(ctx, job.Job{ + Dir: dir, + Func: func(ctx context.Context) error { + return jobs.PreloadEmbeddedSchema(ctx, f.logger, schemas.FS, + f.store, f.stateStore.ProviderSchemas, path) + }, + // DependsOn: none required, since we are inside + Type: operation.OpTypeStacksPreloadEmbeddedSchema.String(), + IgnoreState: ignoreState, + }) + if err != nil { + return deferIds, err + } + deferIds = append(deferIds, eSchemaId) - eSchemaId, err := f.stateStore.JobStore.EnqueueJob(ctx, job.Job{ - Dir: dir, - Func: func(ctx context.Context) error { - return jobs.PreloadEmbeddedSchema(ctx, f.logger, schemas.FS, - f.store, f.stateStore.ProviderSchemas, path) + return deferIds, nil }, - DependsOn: job.IDs{metaId}, - Type: operation.OpTypeStacksPreloadEmbeddedSchema.String(), - IgnoreState: ignoreState, }) if err != nil { return ids, err } - ids = append(ids, eSchemaId) + ids = append(ids, metaId) // TODO: Implement the following functions where appropriate to stacks // Future: decodeDeclaredModuleCalls(ctx, dir, ignoreState) diff --git a/internal/features/stacks/stacks_feature.go b/internal/features/stacks/stacks_feature.go index a197c7601..023aeb416 100644 --- a/internal/features/stacks/stacks_feature.go +++ b/internal/features/stacks/stacks_feature.go @@ -26,9 +26,11 @@ type StacksFeature struct { fs jobs.ReadOnlyFS logger *log.Logger stopFunc context.CancelFunc + + moduleFeature stackDecoder.ModuleReader } -func NewStacksFeature(bus *eventbus.EventBus, stateStore *globalState.StateStore, fs jobs.ReadOnlyFS) (*StacksFeature, error) { +func NewStacksFeature(bus *eventbus.EventBus, stateStore *globalState.StateStore, fs jobs.ReadOnlyFS, moduleFeature stackDecoder.ModuleReader) (*StacksFeature, error) { store, err := state.NewStackStore(stateStore.ChangeStore, stateStore.ProviderSchemas) if err != nil { return nil, err @@ -36,12 +38,13 @@ func NewStacksFeature(bus *eventbus.EventBus, stateStore *globalState.StateStore discardLogger := log.New(io.Discard, "", 0) return &StacksFeature{ - store: store, - bus: bus, - fs: fs, - stateStore: stateStore, - logger: discardLogger, - stopFunc: func() {}, + store: store, + bus: bus, + fs: fs, + stateStore: stateStore, + logger: discardLogger, + stopFunc: func() {}, + moduleFeature: moduleFeature, }, nil } @@ -100,7 +103,8 @@ func (f *StacksFeature) Stop() { func (f *StacksFeature) PathContext(path lang.Path) (*decoder.PathContext, error) { pathReader := &stackDecoder.PathReader{ - StateReader: f.store, + StateReader: f.store, + ModuleReader: f.moduleFeature, } return pathReader.PathContext(path) @@ -108,7 +112,8 @@ func (f *StacksFeature) PathContext(path lang.Path) (*decoder.PathContext, error func (f *StacksFeature) Paths(ctx context.Context) []lang.Path { pathReader := &stackDecoder.PathReader{ - StateReader: f.store, + StateReader: f.store, + ModuleReader: f.moduleFeature, } return pathReader.Paths(ctx) diff --git a/internal/features/stacks/state/stack_store.go b/internal/features/stacks/state/stack_store.go index 896396213..4afc266bc 100644 --- a/internal/features/stacks/state/stack_store.go +++ b/internal/features/stacks/state/stack_store.go @@ -13,6 +13,8 @@ import ( globalState "github.com/hashicorp/terraform-ls/internal/state" globalAst "github.com/hashicorp/terraform-ls/internal/terraform/ast" "github.com/hashicorp/terraform-ls/internal/terraform/module/operation" + tfaddr "github.com/hashicorp/terraform-registry-address" + tfschema "github.com/hashicorp/terraform-schema/schema" tfstack "github.com/hashicorp/terraform-schema/stack" ) @@ -427,3 +429,7 @@ func (s *StackStore) queueRecordChange(oldRecord, newRecord *StackRecord) error return s.changeStore.QueueChange(dir, changes) } + +func (s *StackStore) ProviderSchema(modPath string, addr tfaddr.Provider, vc version.Constraints) (*tfschema.ProviderSchema, error) { + return s.providerSchemasStore.ProviderSchema(modPath, addr, vc) +} diff --git a/internal/langserver/handlers/service.go b/internal/langserver/handlers/service.go index 85be34c55..a22e6758f 100644 --- a/internal/langserver/handlers/service.go +++ b/internal/langserver/handlers/service.go @@ -536,7 +536,7 @@ func (svc *service) configureSessionDependencies(ctx context.Context, cfgOpts *s variablesFeature.SetLogger(svc.logger) variablesFeature.Start(svc.sessCtx) - stacksFeature, err := stacks.NewStacksFeature(svc.eventBus, svc.stateStore, svc.fs) + stacksFeature, err := stacks.NewStacksFeature(svc.eventBus, svc.stateStore, svc.fs, modulesFeature) if err != nil { return err } diff --git a/internal/langserver/handlers/session_mock_test.go b/internal/langserver/handlers/session_mock_test.go index a00beaa5c..5d5571fd5 100644 --- a/internal/langserver/handlers/session_mock_test.go +++ b/internal/langserver/handlers/session_mock_test.go @@ -162,7 +162,7 @@ func NewTestFeatures(eventBus *eventbus.EventBus, s *state.StateStore, fs *files return nil, err } - stacksFeature, err := fstacks.NewStacksFeature(eventBus, s, fs) + stacksFeature, err := fstacks.NewStacksFeature(eventBus, s, fs, modulesFeature) if err != nil { return nil, err }