Index uninitialized modules

go.mod

@@ -19,7 +19,7 @@ require (
 	github.com/hashicorp/terraform-exec v0.17.2
 	github.com/hashicorp/terraform-json v0.14.0
 	github.com/hashicorp/terraform-registry-address v0.0.0-20220623143253-7d51757b572c
-	github.com/hashicorp/terraform-schema v0.0.0-20220708122804-408b2aefd4d6
+	github.com/hashicorp/terraform-schema v0.0.0-20220711140635-cde0aa85508a
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mh-cbon/go-fmt-fail v0.0.0-20160815164508-67765b3fbcb5
 	github.com/mitchellh/cli v1.1.4

go.sum

@@ -344,8 +344,10 @@ github.com/hashicorp/terraform-json v0.14.0 h1:sh9iZ1Y8IFJLx+xQiKHGud6/TSUCM0N8e
 github.com/hashicorp/terraform-json v0.14.0/go.mod h1:5A9HIWPkk4e5aeeXIBbkcOvaZbIYnAIkEyqP2pNSckM=
 github.com/hashicorp/terraform-registry-address v0.0.0-20220623143253-7d51757b572c h1:D8aRO6+mTqHfLsK/BC3j5OAoogv1WLRWzY1AaTo3rBg=
 github.com/hashicorp/terraform-registry-address v0.0.0-20220623143253-7d51757b572c/go.mod h1:Wn3Na71knbXc1G8Lh+yu/dQWWJeFQEpDeJMtWMtlmNI=
-github.com/hashicorp/terraform-schema v0.0.0-20220708122804-408b2aefd4d6 h1:nyYG4twPkXiAUnLSgF5HfhKUGNp3rErv2sYOS2bDsZY=
-github.com/hashicorp/terraform-schema v0.0.0-20220708122804-408b2aefd4d6/go.mod h1:GL+zHwXHrTc/MfKnQP8K2Z4hmaTck85ndiIbvZueMng=
+github.com/hashicorp/terraform-schema v0.0.0-20220711131244-db2cb398f8da h1:Ltvm5bHXN+j18VIMDdwu1qOXzKwe+3Um044ntSMZl8w=
+github.com/hashicorp/terraform-schema v0.0.0-20220711131244-db2cb398f8da/go.mod h1:GL+zHwXHrTc/MfKnQP8K2Z4hmaTck85ndiIbvZueMng=
+github.com/hashicorp/terraform-schema v0.0.0-20220711140635-cde0aa85508a h1:y2Dt1C/FiQy8fLnCKDWipae0uHvfuBmCnG4CHZWRQko=
+github.com/hashicorp/terraform-schema v0.0.0-20220711140635-cde0aa85508a/go.mod h1:GL+zHwXHrTc/MfKnQP8K2Z4hmaTck85ndiIbvZueMng=
 github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734 h1:HKLsbzeOsfXmKNpr3GiT18XAblV0BjCbzL8KQAMZGa0=
 github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734/go.mod h1:kNDNcF7sN4DocDLBkQYz73HGKwN1ANB1blq4lIYLYvg=
 github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=

internal/indexer/module_calls.go

@@ -49,29 +49,24 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) job
 				Type: op.OpTypeParseModuleConfiguration.String(),
 				Defer: func(ctx context.Context, jobErr error) (job.IDs, error) {
 					ids := make(job.IDs, 0)
-					var errs *multierror.Error
 
 					id, err := idx.jobStore.EnqueueJob(job.Job{
 						Dir:  mcHandle,
 						Type: op.OpTypeLoadModuleMetadata.String(),
 						Func: func(ctx context.Context) error {
 							return module.LoadModuleMetadata(idx.modStore, mcPath)
 						},
+						Defer: func(ctx context.Context, jobErr error) (job.IDs, error) {
+							return idx.collectReferences(ctx, mcHandle)
+						},
 					})
 					if err != nil {
-						errs = multierror.Append(errs, err)
+						return ids, err
 					} else {
 						ids = append(ids, id)
 					}
 
-					rIds, err := idx.collectReferences(ctx, mcHandle)
-					if err != nil {
-						errs = multierror.Append(errs, err)
-					} else {
-						ids = append(ids, rIds...)
-					}
-
-					return ids, errs.ErrorOrNil()
+					return ids, nil
 				},
 			})
 			if err != nil {

internal/indexer/walker.go

@@ -16,17 +16,40 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
 	ids := make(job.IDs, 0)
 	var errs *multierror.Error
 
+	// blockingJobIds tracks job IDs which need to finish
+	// prior to collecting references
+	blockingJobIds := make(job.IDs, 0)
+
 	id, err := idx.jobStore.EnqueueJob(job.Job{
 		Dir: modHandle,
 		Func: func(ctx context.Context) error {
 			return module.ParseModuleConfiguration(idx.fs, idx.modStore, modHandle.Path())
 		},
 		Type: op.OpTypeParseModuleConfiguration.String(),
+		Defer: func(ctx context.Context, jobErr error) (job.IDs, error) {
+			ids := make(job.IDs, 0)
+
+			id, err := idx.jobStore.EnqueueJob(job.Job{
+				Dir:  modHandle,
+				Type: op.OpTypeLoadModuleMetadata.String(),
+				Func: func(ctx context.Context) error {
+					return module.LoadModuleMetadata(idx.modStore, modHandle.Path())
+				},
+			})
+			if err != nil {
+				return ids, err
+			} else {
+				ids = append(ids, id)
+			}
+
+			return ids, nil
+		},
 	})
 	if err != nil {
 		errs = multierror.Append(errs, err)
 	} else {
 		ids = append(ids, id)
+		blockingJobIds = append(blockingJobIds, id)
 	}
 
 	id, err = idx.jobStore.EnqueueJob(job.Job{
@@ -70,6 +93,7 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
 		errs = multierror.Append(errs, err)
 	} else {
 		ids = append(ids, id)
+		blockingJobIds = append(blockingJobIds, id)
 	}
 
 	dataDir := datadir.WalkDataDirOfModule(idx.fs, modHandle.Path())
@@ -88,6 +112,7 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
 			errs = multierror.Append(errs, err)
 		} else {
 			ids = append(ids, id)
+			blockingJobIds = append(blockingJobIds, id)
 		}
 	}
 
@@ -106,13 +131,17 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
 			errs = multierror.Append(errs, err)
 		} else {
 			ids = append(ids, id)
+			blockingJobIds = append(blockingJobIds, id)
 		}
+	}
 
-		// Here we wait for all module calls to be processed to
-		// reflect any metadata required to collect reference origins.
-		// This assumes scheduler is running to consume the jobs
-		// by the time we reach this point.
-		idx.jobStore.WaitForJobs(ctx, id)
+	// Here we wait for all dependent jobs to be processed to
+	// reflect any data required to collect reference origins.
+	// This assumes scheduler is running to consume the jobs
+	// by the time we reach this point.
+	err = idx.jobStore.WaitForJobs(ctx, blockingJobIds...)
+	if err != nil {
+		return ids, err
 	}
 
 	rIds, err := idx.collectReferences(ctx, modHandle)

internal/langserver/handlers/complete_test.go

@@ -41,7 +41,7 @@ func TestModuleCompletion_withoutInitialization(t *testing.T) {
 		}`, TempDir(t).URI)}, session.SessionNotInitialized.Err())
 }
 
-func TestModuleCompletion_withValidData(t *testing.T) {
+func TestModuleCompletion_withValidData_basic(t *testing.T) {
 	tmpDir := TempDir(t)
 	InitPluginCache(t, tmpDir.Path())
 

internal/langserver/handlers/did_change_watched_files_test.go

@@ -735,22 +735,22 @@ func TestLangServer_DidChangeWatchedFiles_pluginChange(t *testing.T) {
 					},
 					{
 						Method:        "GetExecPath",
-						Repeatability: 2,
+						Repeatability: 1,
 						ReturnArguments: []interface{}{
 							"",
 						},
 					},
 					{
 						Method:        "ProviderSchemas",
-						Repeatability: 2,
+						Repeatability: 1,
 						Arguments: []interface{}{
 							mock.AnythingOfType(""),
 						},
 						ReturnArguments: []interface{}{
 							&tfjson.ProviderSchemas{
 								FormatVersion: "0.1",
 								Schemas: map[string]*tfjson.ProviderSchema{
-									"test": {
+									"foo": {
 										ConfigSchema: &tfjson.Schema{},
 									},
 								},
@@ -780,12 +780,12 @@ func TestLangServer_DidChangeWatchedFiles_pluginChange(t *testing.T) {
 		ReqParams: "{}",
 	})
 
-	addr := tfaddr.MustParseProviderSource("-/test")
+	addr := tfaddr.MustParseProviderSource("-/foo")
 	vc := version.MustConstraints(version.NewConstraint(">= 1.0"))
 
 	_, err = ss.ProviderSchemas.ProviderSchema(testHandle.Path(), addr, vc)
 	if err == nil {
-		t.Fatal("expected -/test schema to be missing")
+		t.Fatal("expected -/foo schema to be missing")
 	}
 
 	// Install Terraform
@@ -876,7 +876,7 @@ func TestLangServer_DidChangeWatchedFiles_moduleInstalled(t *testing.T) {
 		ReqParams: "{}",
 	})
 
-	submodulePath := filepath.Join(testDir, "application")
+	submodulePath := filepath.Join(testDir, ".terraform", "modules", "azure-hcp-consul")
 	_, err = ss.Modules.ModuleByPath(submodulePath)
 	if err == nil || !state.IsModuleNotFound(err) {
 		t.Fatalf("expected submodule not to be found: %s", err)
@@ -922,7 +922,7 @@ func TestLangServer_DidChangeWatchedFiles_moduleInstalled(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if len(mod.Meta.Variables) != 3 {
-		t.Fatalf("expected exactly 3 variables, %d given", len(mod.Meta.Variables))
+	if len(mod.Meta.Variables) != 8 {
+		t.Fatalf("expected exactly 8 variables, %d given", len(mod.Meta.Variables))
 	}
 }

internal/langserver/handlers/handlers_test.go

@@ -285,4 +285,14 @@ func InitPluginCache(t *testing.T, dir string) {
 	if err != nil {
 		t.Fatal(err)
 	}
+
+	// create an empty file such that it's recognized as an indexable workspace
+	f, err = os.Create(filepath.Join(dir, "empty.tf"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = f.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
 }

internal/langserver/handlers/testdata/uninitialized-single-submodule/main.tf

@@ -1,6 +1,3 @@
-module "gorilla-app" {
-  source           = "./application"
-  environment_name = "prod"
-  app_prefix       = "protect-gorillas"
-  instances        = 5
+module "azure-hcp-consul" {
+  source = "github.com/hashicorp/terraform-azurerm-hcp-consul?ref=v0.2.4"
 }

internal/state/module.go

@@ -73,7 +73,7 @@ func (mm ModuleMetadata) Copy() ModuleMetadata {
 	if mm.ModuleCalls != nil {
 		newMm.ModuleCalls = make(map[string]tfmod.DeclaredModuleCall, len(mm.ModuleCalls))
 		for name, moduleCall := range mm.ModuleCalls {
-			newMm.ModuleCalls[name] = moduleCall
+			newMm.ModuleCalls[name] = moduleCall.Copy()
 		}
 	}
 
@@ -362,6 +362,7 @@ func (s *ModuleStore) ModuleCalls(modPath string) (tfmod.ModuleCalls, error) {
 			LocalName:  mc.LocalName,
 			SourceAddr: mc.SourceAddr,
 			Version:    mc.Version,
+			InputNames: mc.InputNames,
 		}
 	}
 

internal/terraform/ast/module.go

@@ -17,7 +17,7 @@ func (mf ModFilename) IsJSON() bool {
 }
 
 func (mf ModFilename) IsIgnored() bool {
-	return isIgnoredFile(string(mf))
+	return IsIgnoredFile(string(mf))
 }
 
 func IsModuleFilename(name string) bool {
@@ -28,7 +28,7 @@ func IsModuleFilename(name string) bool {
 // isIgnoredFile returns true if the given filename (which must not have a
 // directory path ahead of it) should be ignored as e.g. an editor swap file.
 // See https://github.com/hashicorp/terraform/blob/d35bc05/internal/configs/parser_config_dir.go#L107
-func isIgnoredFile(name string) bool {
+func IsIgnoredFile(name string) bool {
 	return strings.HasPrefix(name, ".") || // Unix-like hidden files
 		strings.HasSuffix(name, "~") || // vim
 		strings.HasPrefix(name, "#") && strings.HasSuffix(name, "#") // emacs

internal/walker/walker.go

@@ -11,7 +11,7 @@ import (
 
 	"github.com/hashicorp/terraform-ls/internal/document"
 	"github.com/hashicorp/terraform-ls/internal/job"
-	"github.com/hashicorp/terraform-ls/internal/terraform/datadir"
+	"github.com/hashicorp/terraform-ls/internal/terraform/ast"
 )
 
 var (
@@ -159,6 +159,8 @@ func (w *Walker) isSkippableDir(dirName string) bool {
 }
 
 func (w *Walker) walk(ctx context.Context, dir document.DirHandle) error {
+	dirsWalked := make(map[string]struct{}, 0)
+
 	err := fs.WalkDir(w.fs, dir.Path(), func(path string, info fs.DirEntry, err error) error {
 		select {
 		case <-ctx.Done():
@@ -186,7 +188,18 @@ func (w *Walker) walk(ctx context.Context, dir document.DirHandle) error {
 			return filepath.SkipDir
 		}
 
-		if info.Name() == datadir.DataDirName {
+		// TODO: replace local map lookup with w.modStore.HasChangedSince(modTime)
+		// once available
+		// See https://github.com/hashicorp/terraform-ls/issues/989
+		_, walked := dirsWalked[dir]
+
+		w.logger.Printf("walker checking file %q; !walked: %t && isModule: %t && !isIgnored: %t",
+			info.Name(),
+			walked, ast.IsModuleFilename(info.Name()), ast.IsIgnoredFile(info.Name()))
+
+		if !walked && ast.IsModuleFilename(info.Name()) && !ast.IsIgnoredFile(info.Name()) {
+			dirsWalked[dir] = struct{}{}
+
 			w.logger.Printf("found module %s", dir)
 
 			exists, err := w.modStore.Exists(dir)
@@ -210,8 +223,8 @@ func (w *Walker) walk(ctx context.Context, dir document.DirHandle) error {
 			return nil
 		}
 
-		if !info.IsDir() {
-			// All files are skipped, we only care about dirs
+		if info.IsDir() {
+			// All other files are skipped
 			return nil
 		}