diff --git a/go.mod b/go.mod index 7fc86b234..462778134 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/hashicorp/terraform-ls -go 1.13 +go 1.16 require ( github.com/agext/levenshtein v1.2.2 // indirect @@ -19,7 +19,7 @@ require ( github.com/hashicorp/terraform-exec v0.17.2 github.com/hashicorp/terraform-json v0.14.0 github.com/hashicorp/terraform-registry-address v0.0.0-20220623143253-7d51757b572c - github.com/hashicorp/terraform-schema v0.0.0-20220708122804-408b2aefd4d6 + github.com/hashicorp/terraform-schema v0.0.0-20220712135911-08c17f04e2bc github.com/kylelemons/godebug v1.1.0 // indirect github.com/mh-cbon/go-fmt-fail v0.0.0-20160815164508-67765b3fbcb5 github.com/mitchellh/cli v1.1.4 diff --git a/go.sum b/go.sum index 9ae2ffac7..008d7cb40 100644 --- a/go.sum +++ b/go.sum @@ -86,7 +86,6 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antonmedv/expr v1.9.0 h1:j4HI3NHEdgDnN9p6oI6Ndr0G5QryMY0FNxT4ONrFDGU= @@ -102,7 +101,6 @@ github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -163,7 +161,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= @@ -175,14 +172,12 @@ github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gizak/termui/v3 v3.1.0/go.mod h1:bXQEBkJpzxUAKf0+xq9MSWAvWZlE7c+aidmyFlkYTrY= -github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34= github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= -github.com/go-git/go-git-fixtures/v4 v4.2.1 h1:n9gGL1Ct/yIw+nfsfr8s4+sbhT+Ncu2SubfXjIWgci8= github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0= github.com/go-git/go-git/v5 v5.4.2 h1:BXyZu9t0VkbiHtqrsvdq39UDhGJTl1h55VW6CSC4aY4= github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc= @@ -344,8 +339,8 @@ github.com/hashicorp/terraform-json v0.14.0 h1:sh9iZ1Y8IFJLx+xQiKHGud6/TSUCM0N8e github.com/hashicorp/terraform-json v0.14.0/go.mod h1:5A9HIWPkk4e5aeeXIBbkcOvaZbIYnAIkEyqP2pNSckM= github.com/hashicorp/terraform-registry-address v0.0.0-20220623143253-7d51757b572c h1:D8aRO6+mTqHfLsK/BC3j5OAoogv1WLRWzY1AaTo3rBg= github.com/hashicorp/terraform-registry-address v0.0.0-20220623143253-7d51757b572c/go.mod h1:Wn3Na71knbXc1G8Lh+yu/dQWWJeFQEpDeJMtWMtlmNI= -github.com/hashicorp/terraform-schema v0.0.0-20220708122804-408b2aefd4d6 h1:nyYG4twPkXiAUnLSgF5HfhKUGNp3rErv2sYOS2bDsZY= -github.com/hashicorp/terraform-schema v0.0.0-20220708122804-408b2aefd4d6/go.mod h1:GL+zHwXHrTc/MfKnQP8K2Z4hmaTck85ndiIbvZueMng= +github.com/hashicorp/terraform-schema v0.0.0-20220712135911-08c17f04e2bc h1:08Pp12m9Rv4cQgk2GOlkoG6njaiAyCR+rKDpdrgvlVs= +github.com/hashicorp/terraform-schema v0.0.0-20220712135911-08c17f04e2bc/go.mod h1:GL+zHwXHrTc/MfKnQP8K2Z4hmaTck85ndiIbvZueMng= github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734 h1:HKLsbzeOsfXmKNpr3GiT18XAblV0BjCbzL8KQAMZGa0= github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734/go.mod h1:kNDNcF7sN4DocDLBkQYz73HGKwN1ANB1blq4lIYLYvg= github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= @@ -405,7 +400,6 @@ github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1 github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -470,7 +464,6 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6 github.com/otiai10/copy v1.7.0 h1:hVoPiN+t+7d2nzzwMiDHPSOogsWAStewq3TwU05+clE= github.com/otiai10/copy v1.7.0/go.mod h1:rmRl6QPdJj6EiUqXQ/4Nn2lLXoNQjFCQbbNrxgc/t3U= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= -github.com/otiai10/curr v1.0.0 h1:TJIWdbX0B+kpNagQrjgq8bCMrbhiuX73M2XwgtDMoOI= github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/otiai10/mint v1.3.3 h1:7JgpsBaN0uMkyju4tbYHu0mnM55hNKVYLsXmwr15NQI= diff --git a/internal/cmd/inspect_module_command.go b/internal/cmd/inspect_module_command.go index 1adef6683..862648164 100644 --- a/internal/cmd/inspect_module_command.go +++ b/internal/cmd/inspect_module_command.go @@ -103,7 +103,7 @@ func (c *InspectModuleCommand) inspect(rootPath string) error { exec.NewExecutor, registry.NewClient()) pa := state.NewPathAwaiter(ss.WalkerPaths, false) - w := walker.NewWalker(pa, ss.Modules, indexer.WalkedModule) + w := walker.NewWalker(fs, pa, ss.Modules, indexer.WalkedModule) w.Collector = walker.NewWalkerCollector() w.SetLogger(c.logger) err = ss.WalkerPaths.WaitForDirs(ctx, []document.DirHandle{dir}) @@ -199,7 +199,7 @@ func formatModuleRecords(mds []datadir.ModuleRecord) []string { continue } if m.IsExternal() { - out = append(out, "EXTERNAL(%s)", m.SourceAddr) + out = append(out, "EXTERNAL(%s)", m.SourceAddr.String()) continue } out = append(out, fmt.Sprintf("%s (%s)", m.Dir, m.SourceAddr)) diff --git a/internal/indexer/module_calls.go b/internal/indexer/module_calls.go index 31315600b..83e544b32 100644 --- a/internal/indexer/module_calls.go +++ b/internal/indexer/module_calls.go @@ -49,7 +49,6 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) job Type: op.OpTypeParseModuleConfiguration.String(), Defer: func(ctx context.Context, jobErr error) (job.IDs, error) { ids := make(job.IDs, 0) - var errs *multierror.Error id, err := idx.jobStore.EnqueueJob(job.Job{ Dir: mcHandle, @@ -57,21 +56,17 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) job Func: func(ctx context.Context) error { return module.LoadModuleMetadata(idx.modStore, mcPath) }, + Defer: func(ctx context.Context, jobErr error) (job.IDs, error) { + return idx.collectReferences(ctx, mcHandle) + }, }) if err != nil { - errs = multierror.Append(errs, err) + return ids, err } else { ids = append(ids, id) } - rIds, err := idx.collectReferences(ctx, mcHandle) - if err != nil { - errs = multierror.Append(errs, err) - } else { - ids = append(ids, rIds...) - } - - return ids, errs.ErrorOrNil() + return ids, nil }, }) if err != nil { diff --git a/internal/indexer/walker.go b/internal/indexer/walker.go index 216172508..354436ece 100644 --- a/internal/indexer/walker.go +++ b/internal/indexer/walker.go @@ -16,17 +16,40 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand ids := make(job.IDs, 0) var errs *multierror.Error + // blockingJobIds tracks job IDs which need to finish + // prior to collecting references + blockingJobIds := make(job.IDs, 0) + id, err := idx.jobStore.EnqueueJob(job.Job{ Dir: modHandle, Func: func(ctx context.Context) error { return module.ParseModuleConfiguration(idx.fs, idx.modStore, modHandle.Path()) }, Type: op.OpTypeParseModuleConfiguration.String(), + Defer: func(ctx context.Context, jobErr error) (job.IDs, error) { + ids := make(job.IDs, 0) + + id, err := idx.jobStore.EnqueueJob(job.Job{ + Dir: modHandle, + Type: op.OpTypeLoadModuleMetadata.String(), + Func: func(ctx context.Context) error { + return module.LoadModuleMetadata(idx.modStore, modHandle.Path()) + }, + }) + if err != nil { + return ids, err + } else { + ids = append(ids, id) + } + + return ids, nil + }, }) if err != nil { errs = multierror.Append(errs, err) } else { ids = append(ids, id) + blockingJobIds = append(blockingJobIds, id) } id, err = idx.jobStore.EnqueueJob(job.Job{ @@ -70,6 +93,7 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand errs = multierror.Append(errs, err) } else { ids = append(ids, id) + blockingJobIds = append(blockingJobIds, id) } dataDir := datadir.WalkDataDirOfModule(idx.fs, modHandle.Path()) @@ -88,6 +112,7 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand errs = multierror.Append(errs, err) } else { ids = append(ids, id) + blockingJobIds = append(blockingJobIds, id) } } @@ -106,13 +131,17 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand errs = multierror.Append(errs, err) } else { ids = append(ids, id) + blockingJobIds = append(blockingJobIds, id) } + } - // Here we wait for all module calls to be processed to - // reflect any metadata required to collect reference origins. - // This assumes scheduler is running to consume the jobs - // by the time we reach this point. - idx.jobStore.WaitForJobs(ctx, id) + // Here we wait for all dependent jobs to be processed to + // reflect any data required to collect reference origins. + // This assumes scheduler is running to consume the jobs + // by the time we reach this point. + err = idx.jobStore.WaitForJobs(ctx, blockingJobIds...) + if err != nil { + return ids, err } rIds, err := idx.collectReferences(ctx, modHandle) diff --git a/internal/langserver/handlers/complete_test.go b/internal/langserver/handlers/complete_test.go index e8a50c421..6d938eeb7 100644 --- a/internal/langserver/handlers/complete_test.go +++ b/internal/langserver/handlers/complete_test.go @@ -41,7 +41,7 @@ func TestModuleCompletion_withoutInitialization(t *testing.T) { }`, TempDir(t).URI)}, session.SessionNotInitialized.Err()) } -func TestModuleCompletion_withValidData(t *testing.T) { +func TestModuleCompletion_withValidData_basic(t *testing.T) { tmpDir := TempDir(t) InitPluginCache(t, tmpDir.Path()) diff --git a/internal/langserver/handlers/did_change_watched_files_test.go b/internal/langserver/handlers/did_change_watched_files_test.go index a983b4f2f..9c2262a9f 100644 --- a/internal/langserver/handlers/did_change_watched_files_test.go +++ b/internal/langserver/handlers/did_change_watched_files_test.go @@ -735,14 +735,14 @@ func TestLangServer_DidChangeWatchedFiles_pluginChange(t *testing.T) { }, { Method: "GetExecPath", - Repeatability: 2, + Repeatability: 1, ReturnArguments: []interface{}{ "", }, }, { Method: "ProviderSchemas", - Repeatability: 2, + Repeatability: 1, Arguments: []interface{}{ mock.AnythingOfType(""), }, @@ -750,7 +750,7 @@ func TestLangServer_DidChangeWatchedFiles_pluginChange(t *testing.T) { &tfjson.ProviderSchemas{ FormatVersion: "0.1", Schemas: map[string]*tfjson.ProviderSchema{ - "test": { + "foo": { ConfigSchema: &tfjson.Schema{}, }, }, @@ -780,12 +780,12 @@ func TestLangServer_DidChangeWatchedFiles_pluginChange(t *testing.T) { ReqParams: "{}", }) - addr := tfaddr.MustParseProviderSource("-/test") + addr := tfaddr.MustParseProviderSource("-/foo") vc := version.MustConstraints(version.NewConstraint(">= 1.0")) _, err = ss.ProviderSchemas.ProviderSchema(testHandle.Path(), addr, vc) if err == nil { - t.Fatal("expected -/test schema to be missing") + t.Fatal("expected -/foo schema to be missing") } // Install Terraform @@ -876,7 +876,7 @@ func TestLangServer_DidChangeWatchedFiles_moduleInstalled(t *testing.T) { ReqParams: "{}", }) - submodulePath := filepath.Join(testDir, "application") + submodulePath := filepath.Join(testDir, ".terraform", "modules", "azure-hcp-consul") _, err = ss.Modules.ModuleByPath(submodulePath) if err == nil || !state.IsModuleNotFound(err) { t.Fatalf("expected submodule not to be found: %s", err) @@ -922,7 +922,7 @@ func TestLangServer_DidChangeWatchedFiles_moduleInstalled(t *testing.T) { t.Fatal(err) } - if len(mod.Meta.Variables) != 3 { - t.Fatalf("expected exactly 3 variables, %d given", len(mod.Meta.Variables)) + if len(mod.Meta.Variables) != 8 { + t.Fatalf("expected exactly 8 variables, %d given", len(mod.Meta.Variables)) } } diff --git a/internal/langserver/handlers/handlers_test.go b/internal/langserver/handlers/handlers_test.go index ac84c556b..2539ce704 100644 --- a/internal/langserver/handlers/handlers_test.go +++ b/internal/langserver/handlers/handlers_test.go @@ -285,4 +285,14 @@ func InitPluginCache(t *testing.T, dir string) { if err != nil { t.Fatal(err) } + + // create an empty file such that it's recognized as an indexable workspace + f, err = os.Create(filepath.Join(dir, "empty.tf")) + if err != nil { + t.Fatal(err) + } + err = f.Close() + if err != nil { + t.Fatal(err) + } } diff --git a/internal/langserver/handlers/service.go b/internal/langserver/handlers/service.go index 7c0c313dc..d0dccdd8e 100644 --- a/internal/langserver/handlers/service.go +++ b/internal/langserver/handlers/service.go @@ -493,12 +493,12 @@ func (svc *service) configureSessionDependencies(ctx context.Context, cfgOpts *s } closedPa := state.NewPathAwaiter(svc.stateStore.WalkerPaths, false) - svc.closedDirWalker = walker.NewWalker(closedPa, svc.modStore, svc.indexer.WalkedModule) + svc.closedDirWalker = walker.NewWalker(svc.fs, closedPa, svc.modStore, svc.indexer.WalkedModule) svc.closedDirWalker.Collector = svc.walkerCollector svc.closedDirWalker.SetLogger(svc.logger) opendPa := state.NewPathAwaiter(svc.stateStore.WalkerPaths, true) - svc.openDirWalker = walker.NewWalker(opendPa, svc.modStore, svc.indexer.WalkedModule) + svc.openDirWalker = walker.NewWalker(svc.fs, opendPa, svc.modStore, svc.indexer.WalkedModule) svc.closedDirWalker.Collector = svc.walkerCollector svc.openDirWalker.SetLogger(svc.logger) diff --git a/internal/langserver/handlers/testdata/uninitialized-single-submodule/application/main.tf b/internal/langserver/handlers/testdata/uninitialized-single-submodule/application/main.tf deleted file mode 100644 index 757a72eab..000000000 --- a/internal/langserver/handlers/testdata/uninitialized-single-submodule/application/main.tf +++ /dev/null @@ -1,18 +0,0 @@ -variable "environment_name" { - type = string -} - -variable "app_prefix" { - type = string -} - -variable "instances" { - type = number -} - -resource "random_pet" "application" { - count = var.instances - keepers = { - unique = "${var.environment_name}-${var.app_prefix}" - } -} diff --git a/internal/langserver/handlers/testdata/uninitialized-single-submodule/main.tf b/internal/langserver/handlers/testdata/uninitialized-single-submodule/main.tf index 6a0bc50e6..d28dd2e2d 100644 --- a/internal/langserver/handlers/testdata/uninitialized-single-submodule/main.tf +++ b/internal/langserver/handlers/testdata/uninitialized-single-submodule/main.tf @@ -1,6 +1,3 @@ -module "gorilla-app" { - source = "./application" - environment_name = "prod" - app_prefix = "protect-gorillas" - instances = 5 +module "azure-hcp-consul" { + source = "github.com/hashicorp/terraform-azurerm-hcp-consul?ref=v0.2.4" } diff --git a/internal/state/module.go b/internal/state/module.go index 2df859da2..6c605530b 100644 --- a/internal/state/module.go +++ b/internal/state/module.go @@ -1,6 +1,7 @@ package state import ( + "fmt" "path/filepath" "github.com/hashicorp/go-memdb" @@ -73,7 +74,7 @@ func (mm ModuleMetadata) Copy() ModuleMetadata { if mm.ModuleCalls != nil { newMm.ModuleCalls = make(map[string]tfmod.DeclaredModuleCall, len(mm.ModuleCalls)) for name, moduleCall := range mm.ModuleCalls { - newMm.ModuleCalls[name] = moduleCall + newMm.ModuleCalls[name] = moduleCall.Copy() } } @@ -362,6 +363,7 @@ func (s *ModuleStore) ModuleCalls(modPath string) (tfmod.ModuleCalls, error) { LocalName: mc.LocalName, SourceAddr: mc.SourceAddr, Version: mc.Version, + InputNames: mc.InputNames, } } @@ -373,6 +375,9 @@ func (s *ModuleStore) LocalModuleMeta(modPath string) (*tfmod.Meta, error) { if err != nil { return nil, err } + if mod.MetaState != op.OpStateLoaded { + return nil, fmt.Errorf("%s: module data not available", modPath) + } return &tfmod.Meta{ Path: mod.Path, ProviderReferences: mod.Meta.ProviderReferences, diff --git a/internal/state/module_test.go b/internal/state/module_test.go index cf674b725..6d1eb0633 100644 --- a/internal/state/module_test.go +++ b/internal/state/module_test.go @@ -92,7 +92,7 @@ func TestModuleStore_CallersOfModule(t *testing.T) { []datadir.ModuleRecord{ { Key: "web_server_sg1", - SourceAddr: "terraform-aws-modules/security-group/aws//modules/http-80", + SourceAddr: tfmod.ParseModuleSourceAddr("terraform-aws-modules/security-group/aws//modules/http-80"), VersionStr: "3.10.0", Version: version.Must(version.NewVersion("3.10.0")), Dir: filepath.Join(".terraform", "modules", "web_server_sg", "terraform-aws-security-group-3.10.0", "modules", "http-80"), @@ -102,7 +102,7 @@ func TestModuleStore_CallersOfModule(t *testing.T) { }, { Key: "local-x", - SourceAddr: "../nested/submodule", + SourceAddr: tfmod.ParseModuleSourceAddr("../nested/submodule"), Dir: filepath.Join("..", "nested", "submodule"), }, }, @@ -115,7 +115,7 @@ func TestModuleStore_CallersOfModule(t *testing.T) { }, { Key: "local-foo", - SourceAddr: "../another/submodule", + SourceAddr: tfmod.ParseModuleSourceAddr("../another/submodule"), Dir: filepath.Join("..", "another", "submodule"), }, }, @@ -125,7 +125,7 @@ func TestModuleStore_CallersOfModule(t *testing.T) { []datadir.ModuleRecord{ { Key: "web_server_sg2", - SourceAddr: "terraform-aws-modules/security-group/aws//modules/http-80", + SourceAddr: tfmod.ParseModuleSourceAddr("terraform-aws-modules/security-group/aws//modules/http-80"), VersionStr: "3.10.0", Version: version.Must(version.NewVersion("3.10.0")), Dir: filepath.Join(".terraform", "modules", "web_server_sg", "terraform-aws-security-group-3.10.0", "modules", "http-80"), @@ -135,7 +135,7 @@ func TestModuleStore_CallersOfModule(t *testing.T) { }, { Key: "local-y", - SourceAddr: "../nested/submodule", + SourceAddr: tfmod.ParseModuleSourceAddr("../nested/submodule"), Dir: filepath.Join("..", "nested", "submodule"), }, }, diff --git a/internal/terraform/ast/module.go b/internal/terraform/ast/module.go index 59d7ebf44..6f26dfebf 100644 --- a/internal/terraform/ast/module.go +++ b/internal/terraform/ast/module.go @@ -17,7 +17,7 @@ func (mf ModFilename) IsJSON() bool { } func (mf ModFilename) IsIgnored() bool { - return isIgnoredFile(string(mf)) + return IsIgnoredFile(string(mf)) } func IsModuleFilename(name string) bool { @@ -28,7 +28,7 @@ func IsModuleFilename(name string) bool { // isIgnoredFile returns true if the given filename (which must not have a // directory path ahead of it) should be ignored as e.g. an editor swap file. // See https://github.com/hashicorp/terraform/blob/d35bc05/internal/configs/parser_config_dir.go#L107 -func isIgnoredFile(name string) bool { +func IsIgnoredFile(name string) bool { return strings.HasPrefix(name, ".") || // Unix-like hidden files strings.HasSuffix(name, "~") || // vim strings.HasPrefix(name, "#") && strings.HasSuffix(name, "#") // emacs diff --git a/internal/terraform/datadir/module_manifest.go b/internal/terraform/datadir/module_manifest.go index 953503c6d..790d1e095 100644 --- a/internal/terraform/datadir/module_manifest.go +++ b/internal/terraform/datadir/module_manifest.go @@ -10,6 +10,7 @@ import ( version "github.com/hashicorp/go-version" "github.com/hashicorp/terraform-ls/internal/pathcmp" + tfmod "github.com/hashicorp/terraform-schema/module" ) func ModuleManifestFilePath(fs fs.StatFS, modulePath string) (string, bool) { @@ -34,11 +35,12 @@ type ModuleRecord struct { // position within the static module tree. Key string `json:"Key"` - // SourceAddr is the source address given for this module in configuration. - // This is used only to detect if the source was changed in configuration - // since the module was last installed, which means that the installer - // must re-install it. - SourceAddr string `json:"Source"` + // SourceAddr is the source address for the module. + SourceAddr tfmod.ModuleSourceAddr `json:"-"` + + // RawSourceAddr is the raw source address for the module + // as it appears in the manifest. + RawSourceAddr string `json:"Source"` // Version is the exact version of the module, which results from parsing // VersionStr. nil for un-versioned modules. @@ -61,6 +63,11 @@ func (r *ModuleRecord) UnmarshalJSON(b []byte) error { if err != nil { return err } + + if record.RawSourceAddr != "" { + record.SourceAddr = tfmod.ParseModuleSourceAddr(record.RawSourceAddr) + } + if record.VersionStr != "" { record.Version, err = version.NewVersion(record.VersionStr) if err != nil { diff --git a/internal/terraform/datadir/module_manifest_test.go b/internal/terraform/datadir/module_manifest_test.go index fb75daedd..6031f0fab 100644 --- a/internal/terraform/datadir/module_manifest_test.go +++ b/internal/terraform/datadir/module_manifest_test.go @@ -8,6 +8,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/hashicorp/go-version" + tfmod "github.com/hashicorp/terraform-schema/module" ) func TestParseModuleManifestFromFile(t *testing.T) { @@ -22,26 +23,29 @@ func TestParseModuleManifestFromFile(t *testing.T) { rootDir: modPath, Records: []ModuleRecord{ { - Key: "web_server_sg1", - SourceAddr: "terraform-aws-modules/security-group/aws//modules/http-80", - VersionStr: "3.10.0", - Version: version.Must(version.NewVersion("3.10.0")), - Dir: filepath.Join(".terraform", "modules", "web_server_sg", "terraform-aws-security-group-3.10.0", "modules", "http-80"), + Key: "web_server_sg1", + SourceAddr: tfmod.ParseModuleSourceAddr("terraform-aws-modules/security-group/aws//modules/http-80"), + RawSourceAddr: "terraform-aws-modules/security-group/aws//modules/http-80", + VersionStr: "3.10.0", + Version: version.Must(version.NewVersion("3.10.0")), + Dir: filepath.Join(".terraform", "modules", "web_server_sg", "terraform-aws-security-group-3.10.0", "modules", "http-80"), }, { - Key: "web_server_sg2", - SourceAddr: "terraform-aws-modules/security-group/aws//modules/http-80", - VersionStr: "3.10.0", - Version: version.Must(version.NewVersion("3.10.0")), - Dir: filepath.Join(".terraform", "modules", "web_server_sg", "terraform-aws-security-group-3.10.0", "modules", "http-80"), + Key: "web_server_sg2", + SourceAddr: tfmod.ParseModuleSourceAddr("terraform-aws-modules/security-group/aws//modules/http-80"), + RawSourceAddr: "terraform-aws-modules/security-group/aws//modules/http-80", + VersionStr: "3.10.0", + Version: version.Must(version.NewVersion("3.10.0")), + Dir: filepath.Join(".terraform", "modules", "web_server_sg", "terraform-aws-security-group-3.10.0", "modules", "http-80"), }, { Dir: ".", }, { - Key: "local", - SourceAddr: "./nested/path", - Dir: filepath.Join("nested", "path"), + Key: "local", + SourceAddr: tfmod.ParseModuleSourceAddr("./nested/path"), + RawSourceAddr: "./nested/path", + Dir: filepath.Join("nested", "path"), }, }, } diff --git a/internal/terraform/datadir/module_manifest_unix_test.go b/internal/terraform/datadir/module_manifest_unix_test.go index a74d313e7..6f8a3ed6d 100644 --- a/internal/terraform/datadir/module_manifest_unix_test.go +++ b/internal/terraform/datadir/module_manifest_unix_test.go @@ -9,6 +9,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/hashicorp/go-version" + tfmod "github.com/hashicorp/terraform-schema/module" ) func TestRecord_UnmarshalJSON_basic(t *testing.T) { @@ -23,11 +24,12 @@ func TestRecord_UnmarshalJSON_basic(t *testing.T) { t.Fatal(err) } expectedRecord := ModuleRecord{ - Key: "web_server_sg", - SourceAddr: "terraform-aws-modules/security-group/aws//modules/http-80", - VersionStr: "3.10.0", - Version: expectedVersion, - Dir: ".terraform/modules/web_server_sg/terraform-aws-security-group-3.10.0/modules/http-80", + Key: "web_server_sg", + SourceAddr: tfmod.ParseModuleSourceAddr("terraform-aws-modules/security-group/aws//modules/http-80"), + RawSourceAddr: "terraform-aws-modules/security-group/aws//modules/http-80", + VersionStr: "3.10.0", + Version: expectedVersion, + Dir: ".terraform/modules/web_server_sg/terraform-aws-security-group-3.10.0/modules/http-80", } if diff := cmp.Diff(expectedRecord, record); diff != "" { t.Fatalf("version mismatch: %s", diff) diff --git a/internal/terraform/datadir/module_manifest_windows_test.go b/internal/terraform/datadir/module_manifest_windows_test.go index b246712b6..65e867752 100644 --- a/internal/terraform/datadir/module_manifest_windows_test.go +++ b/internal/terraform/datadir/module_manifest_windows_test.go @@ -6,6 +6,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/hashicorp/go-version" + tfmod "github.com/hashicorp/terraform-schema/module" ) func TestRecord_UnmarshalJSON_basic(t *testing.T) { @@ -20,11 +21,12 @@ func TestRecord_UnmarshalJSON_basic(t *testing.T) { t.Fatal(err) } expectedRecord := ModuleRecord{ - Key: "web_server_sg", - SourceAddr: "terraform-aws-modules/security-group/aws//modules/http-80", - VersionStr: "3.10.0", - Version: expectedVersion, - Dir: `.terraform\modules\web_server_sg\terraform-aws-security-group-3.10.0\modules\http-80`, + Key: "web_server_sg", + SourceAddr: tfmod.ParseModuleSourceAddr("terraform-aws-modules/security-group/aws//modules/http-80"), + RawSourceAddr: "terraform-aws-modules/security-group/aws//modules/http-80", + VersionStr: "3.10.0", + Version: expectedVersion, + Dir: `.terraform\modules\web_server_sg\terraform-aws-security-group-3.10.0\modules\http-80`, } if diff := cmp.Diff(expectedRecord, record); diff != "" { t.Fatalf("version mismatch: %s", diff) diff --git a/internal/walker/walker.go b/internal/walker/walker.go index 271df0f65..11f32126e 100644 --- a/internal/walker/walker.go +++ b/internal/walker/walker.go @@ -4,14 +4,14 @@ import ( "context" "errors" "fmt" + "io/fs" "io/ioutil" "log" - "os" "path/filepath" "github.com/hashicorp/terraform-ls/internal/document" "github.com/hashicorp/terraform-ls/internal/job" - "github.com/hashicorp/terraform-ls/internal/terraform/datadir" + "github.com/hashicorp/terraform-ls/internal/terraform/ast" ) var ( @@ -33,6 +33,7 @@ var ( type pathToWatch struct{} type Walker struct { + fs fs.ReadDirFS pathStore PathStore modStore ModuleStore @@ -59,8 +60,9 @@ type ModuleStore interface { Add(dir string) error } -func NewWalker(pathStore PathStore, modStore ModuleStore, walkFunc WalkFunc) *Walker { +func NewWalker(fs fs.ReadDirFS, pathStore PathStore, modStore ModuleStore, walkFunc WalkFunc) *Walker { return &Walker{ + fs: fs, pathStore: pathStore, modStore: modStore, walkFunc: walkFunc, @@ -157,10 +159,21 @@ func (w *Walker) isSkippableDir(dirName string) bool { } func (w *Walker) walk(ctx context.Context, dir document.DirHandle) error { - // We ignore the passed FS and instead read straight from OS FS - // because that would require reimplementing filepath.Walk and - // the data directory should never be on the virtual filesystem anyway - err := filepath.Walk(dir.Path(), func(path string, info os.FileInfo, err error) error { + if _, ok := w.excludeModulePaths[dir.Path()]; ok { + w.logger.Printf("skipping walk due to dir being excluded: %s", dir.Path()) + return nil + } + + dirEntries, err := fs.ReadDir(w.fs, dir.Path()) + if err != nil { + w.logger.Printf("reading directory failed: %s: %s", dir.Path(), err) + // fs.ReadDir (or at least the os.ReadDir implementation) returns + // the entries it was able to read before the error, along with the error. + } + + dirIndexed := false + + for _, dirEntry := range dirEntries { select { case <-ctx.Done(): w.logger.Printf("cancelling walk of %s...", dir) @@ -168,56 +181,43 @@ func (w *Walker) walk(ctx context.Context, dir document.DirHandle) error { default: } - if err != nil { - w.logger.Printf("unable to access %s: %s", path, err.Error()) - return nil - } - - dir, err := filepath.Abs(filepath.Dir(path)) - if err != nil { - return err - } - - if w.isSkippableDir(info.Name()) { - w.logger.Printf("skipping %s", path) - return filepath.SkipDir + if w.isSkippableDir(dirEntry.Name()) { + w.logger.Printf("skipping ignored dir name: %s", dirEntry.Name()) + continue } - if _, ok := w.excludeModulePaths[dir]; ok { - return filepath.SkipDir - } - - if info.Name() == datadir.DataDirName { + if !dirIndexed && ast.IsModuleFilename(dirEntry.Name()) && !ast.IsIgnoredFile(dirEntry.Name()) { + dirIndexed = true w.logger.Printf("found module %s", dir) - exists, err := w.modStore.Exists(dir) + exists, err := w.modStore.Exists(dir.Path()) if err != nil { return err } if !exists { - err := w.modStore.Add(dir) + err := w.modStore.Add(dir.Path()) if err != nil { return err } } - modHandle := document.DirHandleFromPath(dir) - ids, err := w.walkFunc(ctx, modHandle) + ids, err := w.walkFunc(ctx, dir) if err != nil { w.collectError(err) } w.collectJobIds(ids) - - return nil + continue } - if !info.IsDir() { - // All files are skipped, we only care about dirs - return nil + if dirEntry.IsDir() { + path := filepath.Join(dir.Path(), dirEntry.Name()) + dirHandle := document.DirHandleFromPath(path) + err = w.walk(ctx, dirHandle) + if err != nil { + return err + } } - - return nil - }) + } w.logger.Printf("walking of %s finished", dir) return err } diff --git a/internal/walker/walker_test.go b/internal/walker/walker_test.go index da4bff23d..378d57d79 100644 --- a/internal/walker/walker_test.go +++ b/internal/walker/walker_test.go @@ -23,6 +23,53 @@ import ( "github.com/stretchr/testify/mock" ) +func TestWalker_basic(t *testing.T) { + ss, err := state.NewStateStore() + if err != nil { + t.Fatal(err) + } + + fs := filesystem.NewFilesystem(ss.DocumentStore) + pa := state.NewPathAwaiter(ss.WalkerPaths, false) + + walkFunc := func(ctx context.Context, modHandle document.DirHandle) (job.IDs, error) { + return job.IDs{}, nil + } + + w := NewWalker(fs, pa, ss.Modules, walkFunc) + w.Collector = NewWalkerCollector() + w.SetLogger(testLogger()) + + root, err := filepath.Abs(filepath.Join("testdata", "uninitialized-root")) + if err != nil { + t.Fatal(err) + } + dir := document.DirHandleFromPath(root) + + err = ss.WalkerPaths.EnqueueDir(dir) + if err != nil { + t.Fatal(err) + } + + ctx := context.Background() + err = w.StartWalking(ctx) + if err != nil { + t.Fatal(err) + } + err = ss.WalkerPaths.WaitForDirs(ctx, []document.DirHandle{dir}) + if err != nil { + t.Fatal(err) + } + err = ss.JobStore.WaitForJobs(ctx, w.Collector.JobIds()...) + if err != nil { + t.Fatal(err) + } + err = w.Collector.ErrorOrNil() + if err != nil { + t.Fatal(err) + } +} + func TestWalker_complexModules(t *testing.T) { testData, err := filepath.Abs("testdata") if err != nil { @@ -42,13 +89,63 @@ func TestWalker_complexModules(t *testing.T) { []string{ filepath.Join(testData, "single-root-ext-modules-only"), filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "codelabs", "simple"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "examples", "delete_default_gateway_routes"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "examples", "ilb_routing"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "examples", "multi_vpc"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "examples", "secondary_ranges"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "examples", "simple_project"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "examples", "simple_project_with_regional_network"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "examples", "submodule_firewall"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "examples", "submodule_network_peering"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "examples", "submodule_svpc_access"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "modules", "fabric-net-firewall"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "modules", "fabric-net-svpc-access"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "modules", "network-peering"), filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "modules", "routes"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "modules", "routes-beta"), filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "modules", "subnets"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "modules", "subnets-beta"), filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "modules", "vpc"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "test", "fixtures", "all_examples"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "test", "fixtures", "delete_default_gateway_routes"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "test", "fixtures", "ilb_routing"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "test", "fixtures", "multi_vpc"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "test", "fixtures", "secondary_ranges"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "test", "fixtures", "simple_project"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "test", "fixtures", "simple_project_with_regional_network"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "test", "fixtures", "submodule_firewall"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "test", "fixtures", "submodule_network_peering"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc1", "terraform-google-network-2.3.0", "test", "setup"), filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "codelabs", "simple"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "examples", "delete_default_gateway_routes"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "examples", "ilb_routing"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "examples", "multi_vpc"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "examples", "secondary_ranges"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "examples", "simple_project"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "examples", "simple_project_with_regional_network"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "examples", "submodule_firewall"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "examples", "submodule_network_peering"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "examples", "submodule_svpc_access"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "modules", "fabric-net-firewall"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "modules", "fabric-net-svpc-access"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "modules", "network-peering"), filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "modules", "routes"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "modules", "routes-beta"), filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "modules", "subnets"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "modules", "subnets-beta"), filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "modules", "vpc"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "test", "fixtures", "all_examples"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "test", "fixtures", "delete_default_gateway_routes"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "test", "fixtures", "ilb_routing"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "test", "fixtures", "multi_vpc"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "test", "fixtures", "secondary_ranges"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "test", "fixtures", "simple_project"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "test", "fixtures", "simple_project_with_regional_network"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "test", "fixtures", "submodule_firewall"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "test", "fixtures", "submodule_network_peering"), + filepath.Join(testData, "single-root-ext-modules-only", ".terraform", "modules", "vpc2", "terraform-google-network-2.3.0", "test", "setup"), }, []string{ filepath.Join(testData, "single-root-ext-modules-only"), @@ -61,15 +158,66 @@ func TestWalker_complexModules(t *testing.T) { []string{ filepath.Join(testData, "single-root-local-and-ext-modules"), filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "codelabs", "simple"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "examples", "delete_default_gateway_routes"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "examples", "ilb_routing"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "examples", "multi_vpc"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "examples", "secondary_ranges"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "examples", "simple_project"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "examples", "simple_project_with_regional_network"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "examples", "submodule_firewall"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "examples", "submodule_network_peering"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "examples", "submodule_svpc_access"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "modules", "fabric-net-firewall"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "modules", "fabric-net-svpc-access"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "modules", "network-peering"), filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "modules", "routes"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "modules", "routes-beta"), filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "modules", "subnets"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "modules", "subnets-beta"), filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "modules", "vpc"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "test", "fixtures", "all_examples"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "test", "fixtures", "delete_default_gateway_routes"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "test", "fixtures", "ilb_routing"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "test", "fixtures", "multi_vpc"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "test", "fixtures", "secondary_ranges"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "test", "fixtures", "simple_project"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "test", "fixtures", "simple_project_with_regional_network"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "test", "fixtures", "submodule_firewall"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "test", "fixtures", "submodule_network_peering"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "five", "terraform-google-network-2.3.0", "test", "setup"), filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "codelabs", "simple"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "examples", "delete_default_gateway_routes"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "examples", "ilb_routing"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "examples", "multi_vpc"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "examples", "secondary_ranges"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "examples", "simple_project"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "examples", "simple_project_with_regional_network"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "examples", "submodule_firewall"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "examples", "submodule_network_peering"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "examples", "submodule_svpc_access"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "modules", "fabric-net-firewall"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "modules", "fabric-net-svpc-access"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "modules", "network-peering"), filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "modules", "routes"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "modules", "routes-beta"), filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "modules", "subnets"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "modules", "subnets-beta"), filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "modules", "vpc"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "test", "fixtures", "all_examples"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "test", "fixtures", "delete_default_gateway_routes"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "test", "fixtures", "ilb_routing"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "test", "fixtures", "multi_vpc"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "test", "fixtures", "secondary_ranges"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "test", "fixtures", "simple_project"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "test", "fixtures", "simple_project_with_regional_network"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "test", "fixtures", "submodule_firewall"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "test", "fixtures", "submodule_network_peering"), + filepath.Join(testData, "single-root-local-and-ext-modules", ".terraform", "modules", "four", "terraform-google-network-2.3.0", "test", "setup"), filepath.Join(testData, "single-root-local-and-ext-modules", "alpha"), filepath.Join(testData, "single-root-local-and-ext-modules", "beta"), + filepath.Join(testData, "single-root-local-and-ext-modules", "charlie"), }, []string{ filepath.Join(testData, "single-root-local-and-ext-modules"), @@ -83,6 +231,7 @@ func TestWalker_complexModules(t *testing.T) { filepath.Join(testData, "single-root-local-modules-only"), filepath.Join(testData, "single-root-local-modules-only", "alpha"), filepath.Join(testData, "single-root-local-modules-only", "beta"), + filepath.Join(testData, "single-root-local-modules-only", "charlie"), }, []string{ filepath.Join(testData, "single-root-local-modules-only"), @@ -117,6 +266,7 @@ func TestWalker_complexModules(t *testing.T) { []string{ filepath.Join(testData, "nested-single-root-local-modules-down", "tf-root"), filepath.Join(testData, "nested-single-root-local-modules-down", "tf-root", "alpha"), + filepath.Join(testData, "nested-single-root-local-modules-down", "tf-root", "beta"), filepath.Join(testData, "nested-single-root-local-modules-down", "tf-root", "charlie"), }, []string{ @@ -177,12 +327,15 @@ func TestWalker_complexModules(t *testing.T) { []string{ filepath.Join(testData, "multi-root-local-modules-down", "first-root"), filepath.Join(testData, "multi-root-local-modules-down", "first-root", "alpha"), + filepath.Join(testData, "multi-root-local-modules-down", "first-root", "beta"), filepath.Join(testData, "multi-root-local-modules-down", "first-root", "charlie"), filepath.Join(testData, "multi-root-local-modules-down", "second-root"), filepath.Join(testData, "multi-root-local-modules-down", "second-root", "alpha"), + filepath.Join(testData, "multi-root-local-modules-down", "second-root", "beta"), filepath.Join(testData, "multi-root-local-modules-down", "second-root", "charlie"), filepath.Join(testData, "multi-root-local-modules-down", "third-root"), filepath.Join(testData, "multi-root-local-modules-down", "third-root", "alpha"), + filepath.Join(testData, "multi-root-local-modules-down", "third-root", "beta"), filepath.Join(testData, "multi-root-local-modules-down", "third-root", "charlie"), }, []string{ @@ -234,7 +387,7 @@ func TestWalker_complexModules(t *testing.T) { pa := state.NewPathAwaiter(ss.WalkerPaths, false) indexer := indexer.NewIndexer(fs, ss.Modules, ss.ProviderSchemas, ss.RegistryModules, ss.JobStore, exec.NewMockExecutor(tfCalls), registry.NewClient()) - w := NewWalker(pa, ss.Modules, indexer.WalkedModule) + w := NewWalker(fs, pa, ss.Modules, indexer.WalkedModule) w.Collector = NewWalkerCollector() w.SetLogger(testLogger()) dir := document.DirHandleFromPath(tc.root) @@ -308,8 +461,8 @@ func localProviderSchemaPaths(t *testing.T, it *state.ProviderSchemaIterator) [] func validTfMockCalls(repeatability int) []*mock.Call { return []*mock.Call{ { - Method: "Version", - Repeatability: repeatability, + Method: "Version", + // Repeatability: repeatability, Arguments: []interface{}{ mock.AnythingOfType("*context.valueCtx"), }, @@ -320,15 +473,15 @@ func validTfMockCalls(repeatability int) []*mock.Call { }, }, { - Method: "GetExecPath", - Repeatability: repeatability, + Method: "GetExecPath", + // Repeatability: repeatability, ReturnArguments: []interface{}{ "", }, }, { - Method: "ProviderSchemas", - Repeatability: repeatability, + Method: "ProviderSchemas", + // Repeatability: repeatability, Arguments: []interface{}{ mock.AnythingOfType("*context.valueCtx"), },