diff --git a/internal/cmd/completion_command.go b/internal/cmd/completion_command.go index 0d428df6b..4c3717e85 100644 --- a/internal/cmd/completion_command.go +++ b/internal/cmd/completion_command.go @@ -102,21 +102,27 @@ func (c *CompletionCommand) Run(args []string) int { return 1 } - module, err := module.NewModule(context.Background(), fs, fh.Dir()) + ctx := context.Background() + modMgr := module.NewSyncModuleManager(ctx, fs) + + mod, err := modMgr.AddModule(fh.Dir()) if err != nil { - c.Ui.Error(fmt.Sprintf("failed to load module: %s", err.Error())) + c.Ui.Error(err.Error()) return 1 } - schema, err := module.MergedSchema() + + schema, err := modMgr.SchemaForModule(fh.Dir()) if err != nil { c.Ui.Error(fmt.Sprintf("failed to find schema: %s", err.Error())) return 1 } - d, err := module.DecoderWithSchema(schema) + + d, err := module.DecoderForModule(mod) if err != nil { - c.Ui.Error(fmt.Sprintf("failed to find parser: %s", err.Error())) + c.Ui.Error(fmt.Sprintf("failed to find decoder: %s", err.Error())) return 1 } + d.SetSchema(schema) pos := fPos.Position() diff --git a/internal/cmd/inspect_module_command.go b/internal/cmd/inspect_module_command.go index dad97446c..22b7ddfbe 100644 --- a/internal/cmd/inspect_module_command.go +++ b/internal/cmd/inspect_module_command.go @@ -16,6 +16,7 @@ import ( ictx "github.com/hashicorp/terraform-ls/internal/context" "github.com/hashicorp/terraform-ls/internal/filesystem" "github.com/hashicorp/terraform-ls/internal/logging" + "github.com/hashicorp/terraform-ls/internal/terraform/datadir" "github.com/hashicorp/terraform-ls/internal/terraform/module" "github.com/mitchellh/cli" ) @@ -84,46 +85,50 @@ func (c *InspectModuleCommand) inspect(rootPath string) error { fs := filesystem.NewFilesystem() - modMgr := module.NewModuleManager(fs) + ctx := context.Background() + modMgr := module.NewSyncModuleManager(ctx, fs) modMgr.SetLogger(c.logger) - walker := module.NewWalker() + + walker := module.SyncWalker(fs, modMgr) walker.SetLogger(c.logger) ctx, cancel := ictx.WithSignalCancel(context.Background(), c.logger, syscall.SIGINT, syscall.SIGTERM) defer cancel() - err = walker.StartWalking(ctx, rootPath, func(ctx context.Context, dir string) error { - mod, err := modMgr.AddAndStartLoadingModule(ctx, dir) - if err != nil { - return err - } - <-mod.LoadingDone() - - return nil - }) + err = walker.StartWalking(ctx, rootPath) if err != nil { return err } - <-walker.Done() - modules := modMgr.ListModules() c.Ui.Output(fmt.Sprintf("%d modules found in total at %s", len(modules), rootPath)) for _, mod := range modules { errs := &multierror.Error{} - err := mod.LoadError() + _, err = mod.TerraformVersion() if err != nil { - var ok bool - errs, ok = err.(*multierror.Error) - if !ok { - return err - } + multierror.Append(errs, err) + } + + _, err := mod.ProviderSchema() + if err != nil { + multierror.Append(errs, err) + } + + _, err = mod.ModuleManifest() + if err != nil { + multierror.Append(errs, err) } + + _, err = mod.ParsedFiles() + if err != nil { + multierror.Append(errs, err) + } + errs.ErrorFormat = formatErrors - modules := formatModuleRecords(mod.Modules()) + modules := formatModuleRecords(mod.ModuleCalls()) subModules := fmt.Sprintf("%d modules", len(modules)) if len(modules) > 0 { subModules += "\n" @@ -153,7 +158,7 @@ func formatErrors(errors []error) string { return strings.TrimSpace(out) } -func formatModuleRecords(mds []module.ModuleRecord) []string { +func formatModuleRecords(mds []datadir.ModuleRecord) []string { out := make([]string, 0) for _, m := range mds { if m.IsRoot() { diff --git a/internal/context/context.go b/internal/context/context.go index 753eb4fa5..8f8a65d8b 100644 --- a/internal/context/context.go +++ b/internal/context/context.go @@ -9,7 +9,6 @@ import ( lsp "github.com/hashicorp/terraform-ls/internal/protocol" "github.com/hashicorp/terraform-ls/internal/settings" "github.com/hashicorp/terraform-ls/internal/terraform/module" - "github.com/hashicorp/terraform-ls/internal/watcher" ) type contextKey struct { @@ -29,10 +28,8 @@ var ( ctxTfExecTimeout = &contextKey{"terraform execution timeout"} ctxWatcher = &contextKey{"watcher"} ctxModuleMngr = &contextKey{"module manager"} - ctxTfFormatterFinder = &contextKey{"terraform formatter finder"} - ctxModuleCaFi = &contextKey{"module candidate finder"} + ctxModuleFinder = &contextKey{"module finder"} ctxModuleWalker = &contextKey{"module walker"} - ctxModuleLoader = &contextKey{"module loader"} ctxRootDir = &contextKey{"root directory"} ctxCommandPrefix = &contextKey{"command prefix"} ctxDiags = &contextKey{"diagnostics"} @@ -103,12 +100,12 @@ func TerraformExecTimeout(ctx context.Context) (time.Duration, bool) { return path, ok } -func WithWatcher(ctx context.Context, w watcher.Watcher) context.Context { +func WithWatcher(ctx context.Context, w module.Watcher) context.Context { return context.WithValue(ctx, ctxWatcher, w) } -func Watcher(ctx context.Context) (watcher.Watcher, error) { - w, ok := ctx.Value(ctxWatcher).(watcher.Watcher) +func Watcher(ctx context.Context) (module.Watcher, error) { + w, ok := ctx.Value(ctxWatcher).(module.Watcher) if !ok { return nil, missingContextErr(ctxWatcher) } @@ -127,18 +124,6 @@ func ModuleManager(ctx context.Context) (module.ModuleManager, error) { return wm, nil } -func WithTerraformFormatterFinder(ctx context.Context, tef module.TerraformFormatterFinder) context.Context { - return context.WithValue(ctx, ctxTfFormatterFinder, tef) -} - -func TerraformFormatterFinder(ctx context.Context) (module.TerraformFormatterFinder, error) { - pf, ok := ctx.Value(ctxTfFormatterFinder).(module.TerraformFormatterFinder) - if !ok { - return nil, missingContextErr(ctxTfFormatterFinder) - } - return pf, nil -} - func WithTerraformExecPath(ctx context.Context, path string) context.Context { return context.WithValue(ctx, ctxTfExecPath, path) } @@ -149,13 +134,13 @@ func TerraformExecPath(ctx context.Context) (string, bool) { } func WithModuleFinder(ctx context.Context, mf module.ModuleFinder) context.Context { - return context.WithValue(ctx, ctxModuleCaFi, mf) + return context.WithValue(ctx, ctxModuleFinder, mf) } func ModuleFinder(ctx context.Context) (module.ModuleFinder, error) { - cf, ok := ctx.Value(ctxModuleCaFi).(module.ModuleFinder) + cf, ok := ctx.Value(ctxModuleFinder).(module.ModuleFinder) if !ok { - return nil, missingContextErr(ctxModuleCaFi) + return nil, missingContextErr(ctxModuleFinder) } return cf, nil } @@ -216,18 +201,6 @@ func ModuleWalker(ctx context.Context) (*module.Walker, error) { return w, nil } -func WithModuleLoader(ctx context.Context, ml module.ModuleLoader) context.Context { - return context.WithValue(ctx, ctxModuleLoader, ml) -} - -func ModuleLoader(ctx context.Context) (module.ModuleLoader, error) { - w, ok := ctx.Value(ctxModuleLoader).(module.ModuleLoader) - if !ok { - return nil, missingContextErr(ctxModuleLoader) - } - return w, nil -} - func WithDiagnostics(ctx context.Context, diags *diagnostics.Notifier) context.Context { return context.WithValue(ctx, ctxDiags, diags) } diff --git a/internal/filesystem/filesystem.go b/internal/filesystem/filesystem.go index b564beee2..0d89922a1 100644 --- a/internal/filesystem/filesystem.go +++ b/internal/filesystem/filesystem.go @@ -2,6 +2,7 @@ package filesystem import ( "bytes" + "fmt" "io/ioutil" "log" "os" @@ -36,6 +37,18 @@ func (fs *fsystem) SetLogger(logger *log.Logger) { } func (fs *fsystem) CreateDocument(dh DocumentHandler, text []byte) error { + _, err := fs.memFs.Stat(dh.Dir()) + if err != nil { + if os.IsNotExist(err) { + err := fs.memFs.MkdirAll(dh.Dir(), 0755) + if err != nil { + return fmt.Errorf("failed to create parent dir: %w", err) + } + } else { + return err + } + } + f, err := fs.memFs.Create(dh.FullPath()) if err != nil { return err @@ -196,11 +209,11 @@ func (fs *fsystem) ReadFile(name string) ([]byte, error) { func (fs *fsystem) ReadDir(name string) ([]os.FileInfo, error) { memList, err := afero.ReadDir(fs.memFs, name) if err != nil && !os.IsNotExist(err) { - return nil, err + return nil, fmt.Errorf("memory FS: %w", err) } osList, err := afero.ReadDir(fs.osFs, name) - if err != nil { - return nil, err + if err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("OS FS: %w", err) } list := memList @@ -231,3 +244,12 @@ func (fs *fsystem) Open(name string) (File, error) { return f, err } + +func (fs *fsystem) Stat(name string) (os.FileInfo, error) { + fi, err := fs.memFs.Stat(name) + if err != nil && os.IsNotExist(err) { + return fs.osFs.Stat(name) + } + + return fi, err +} diff --git a/internal/filesystem/filesystem_metadata.go b/internal/filesystem/filesystem_metadata.go index b4166b7d0..ccad86eec 100644 --- a/internal/filesystem/filesystem_metadata.go +++ b/internal/filesystem/filesystem_metadata.go @@ -1,7 +1,10 @@ package filesystem import ( + "path/filepath" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/terraform-ls/internal/uri" ) func (fs *fsystem) markDocumentAsOpen(dh DocumentHandler) error { @@ -16,6 +19,26 @@ func (fs *fsystem) markDocumentAsOpen(dh DocumentHandler) error { return nil } +func (fs *fsystem) HasOpenFiles(dirPath string) (bool, error) { + files, err := fs.ReadDir(dirPath) + if err != nil { + return false, err + } + + fs.docMetaMu.RLock() + defer fs.docMetaMu.RUnlock() + + for _, fi := range files { + u := uri.FromPath(filepath.Join(dirPath, fi.Name())) + dm, ok := fs.docMeta[u] + if ok && dm.IsOpen() { + return true, nil + } + } + + return false, nil +} + func (fs *fsystem) createDocumentMetadata(dh DocumentHandler, text []byte) error { if fs.documentMetadataExists(dh) { return &MetadataAlreadyExistsErr{dh} diff --git a/internal/filesystem/filesystem_test.go b/internal/filesystem/filesystem_test.go index bdbe9ddb9..eae40a5fb 100644 --- a/internal/filesystem/filesystem_test.go +++ b/internal/filesystem/filesystem_test.go @@ -294,6 +294,29 @@ func TestFilesystem_ReadDir(t *testing.T) { } } +func TestFilesystem_ReadDir_memFsOnly(t *testing.T) { + fs := NewFilesystem() + + tmpDir := t.TempDir() + + fh := testHandlerFromPath(filepath.Join(tmpDir, "memfile")) + err := fs.CreateDocument(fh, []byte("test")) + if err != nil { + t.Fatal(err) + } + + fis, err := fs.ReadDir(tmpDir) + if err != nil { + t.Fatal(err) + } + + expectedFis := []string{"memfile"} + names := namesFromFileInfos(fis) + if diff := cmp.Diff(expectedFis, names); diff != "" { + t.Fatalf("file list mismatch: %s", diff) + } +} + func namesFromFileInfos(fis []os.FileInfo) []string { names := make([]string, len(fis), len(fis)) for i, fi := range fis { @@ -433,6 +456,86 @@ func TestFilesystem_Create_memOnly(t *testing.T) { } } +func TestFilesystem_CreateDocument_missingParentDir(t *testing.T) { + fs := NewFilesystem() + + tmpDir := t.TempDir() + testPath := filepath.Join(tmpDir, "foo", "bar", "test.tf") + fh := testHandlerFromPath(testPath) + + err := fs.CreateDocument(fh, []byte("test")) + if err != nil { + t.Fatal(err) + } + + fooPath := filepath.Join(tmpDir, "foo") + fi, err := fs.memFs.Stat(fooPath) + if err != nil { + t.Fatal(err) + } + if !fi.IsDir() { + t.Fatalf("expected %q to be a dir", fooPath) + } + + barPath := filepath.Join(tmpDir, "foo", "bar") + fi, err = fs.memFs.Stat(barPath) + if err != nil { + t.Fatal(err) + } + if !fi.IsDir() { + t.Fatalf("expected %q to be a dir", barPath) + } +} + +func TestFilesystem_HasOpenFiles(t *testing.T) { + fs := NewFilesystem() + + tmpDir := t.TempDir() + + notOpenHandler := filepath.Join(tmpDir, "not-open.tf") + noFh := testHandlerFromPath(notOpenHandler) + err := fs.CreateDocument(noFh, []byte("test1")) + if err != nil { + t.Fatal(err) + } + + of, err := fs.HasOpenFiles(tmpDir) + if err != nil { + t.Fatal(err) + } + if of { + t.Fatalf("expected no open files for %s", tmpDir) + } + + openHandler := filepath.Join(tmpDir, "open.tf") + ofh := testHandlerFromPath(openHandler) + err = fs.CreateAndOpenDocument(ofh, []byte("test2")) + if err != nil { + t.Fatal(err) + } + + of, err = fs.HasOpenFiles(tmpDir) + if err != nil { + t.Fatal(err) + } + if !of { + t.Fatalf("expected open files for %s", tmpDir) + } + + err = fs.CloseAndRemoveDocument(ofh) + if err != nil { + t.Fatal(err) + } + + of, err = fs.HasOpenFiles(tmpDir) + if err != nil { + t.Fatal(err) + } + if of { + t.Fatalf("expected no open files for %s", tmpDir) + } +} + func TempDir(t *testing.T) string { tmpDir := filepath.Join(os.TempDir(), "terraform-ls", t.Name()) @@ -478,6 +581,11 @@ func (fh *testHandler) Dir() string { func (fh *testHandler) Filename() string { return "" } + +func (fh *testHandler) IsOpen() bool { + return false +} + func (fh *testHandler) Version() int { return 0 } diff --git a/internal/filesystem/types.go b/internal/filesystem/types.go index cc9b9e77f..f8df53cf1 100644 --- a/internal/filesystem/types.go +++ b/internal/filesystem/types.go @@ -40,6 +40,7 @@ type DocumentStorage interface { GetDocument(DocumentHandler) (Document, error) CloseAndRemoveDocument(DocumentHandler) error ChangeDocument(VersionedDocumentHandler, DocumentChanges) error + HasOpenFiles(path string) (bool, error) } type Filesystem interface { @@ -51,6 +52,7 @@ type Filesystem interface { ReadFile(name string) ([]byte, error) ReadDir(name string) ([]os.FileInfo, error) Open(name string) (File, error) + Stat(name string) (os.FileInfo, error) } // File represents an open file in FS diff --git a/internal/langserver/diagnostics/validate_diags.go b/internal/langserver/diagnostics/validate_diags.go new file mode 100644 index 000000000..c7564ed52 --- /dev/null +++ b/internal/langserver/diagnostics/validate_diags.go @@ -0,0 +1,45 @@ +package diagnostics + +import ( + "github.com/hashicorp/hcl/v2" + tfjson "github.com/hashicorp/terraform-json" +) + +// tfjson.Diagnostic is a conversion of an internal diag to terraform core, +// tfdiags, which is effectively based on hcl.Diagnostic. +// This process is really just converting it back to hcl.Diagnotic +// since it is the defacto diagnostic type for our codebase currently +// https://github.com/hashicorp/terraform/blob/ae025248cc0712bf53c675dc2fe77af4276dd5cc/command/validate.go#L138 +func HCLDiagsFromJSON(jsonDiags []tfjson.Diagnostic) map[string]hcl.Diagnostics { + diagsMap := make(map[string]hcl.Diagnostics) + + for _, d := range jsonDiags { + // the diagnostic must be tied to a file to exist in the map + if d.Range == nil || d.Range.Filename == "" { + continue + } + + diags := diagsMap[d.Range.Filename] + + var severity hcl.DiagnosticSeverity + if d.Severity == "error" { + severity = hcl.DiagError + } else if d.Severity == "warning" { + severity = hcl.DiagWarning + } + + diags = append(diags, &hcl.Diagnostic{ + Severity: severity, + Summary: d.Summary, + Detail: d.Detail, + Subject: &hcl.Range{ + Filename: d.Range.Filename, + Start: hcl.Pos(d.Range.Start), + End: hcl.Pos(d.Range.End), + }, + }) + diagsMap[d.Range.Filename] = diags + } + + return diagsMap +} diff --git a/internal/langserver/handlers/command/init.go b/internal/langserver/handlers/command/init.go index 84e7861b3..6714f50e1 100644 --- a/internal/langserver/handlers/command/init.go +++ b/internal/langserver/handlers/command/init.go @@ -10,6 +10,7 @@ import ( "github.com/hashicorp/terraform-ls/internal/langserver/progress" ilsp "github.com/hashicorp/terraform-ls/internal/lsp" lsp "github.com/hashicorp/terraform-ls/internal/protocol" + "github.com/hashicorp/terraform-ls/internal/terraform/module" ) func TerraformInitHandler(ctx context.Context, args cmd.CommandArgs) (interface{}, error) { @@ -20,12 +21,24 @@ func TerraformInitHandler(ctx context.Context, args cmd.CommandArgs) (interface{ dh := ilsp.FileHandlerFromDirURI(lsp.DocumentURI(dirUri)) - cf, err := lsctx.ModuleFinder(ctx) + modMgr, err := lsctx.ModuleManager(ctx) if err != nil { return nil, err } - mod, err := cf.ModuleByPath(dh.Dir()) + mod, err := modMgr.ModuleByPath(dh.Dir()) + if err != nil { + if module.IsModuleNotFound(err) { + mod, err = modMgr.AddModule(dh.Dir()) + if err != nil { + return nil, err + } + } else { + return nil, err + } + } + + tfExec, err := module.TerraformExecutorForModule(ctx, mod) if err != nil { return nil, err } @@ -36,22 +49,10 @@ func TerraformInitHandler(ctx context.Context, args cmd.CommandArgs) (interface{ }() progress.Report(ctx, "Running terraform init ...") - err = mod.ExecuteTerraformInit(ctx) - if err != nil { - return nil, err - } - - progress.Report(ctx, "Detecting paths to watch ...") - paths := mod.PathsToWatch() - - w, err := lsctx.Watcher(ctx) + err = tfExec.Init(ctx) if err != nil { return nil, err } - err = w.AddPaths(paths) - if err != nil { - return nil, fmt.Errorf("failed to add watch for dir (%s): %+v", dh.Dir(), err) - } return nil, nil } diff --git a/internal/langserver/handlers/command/modules.go b/internal/langserver/handlers/command/modules.go index a8ccbb62b..56cb085ae 100644 --- a/internal/langserver/handlers/command/modules.go +++ b/internal/langserver/handlers/command/modules.go @@ -10,6 +10,7 @@ import ( "github.com/hashicorp/terraform-ls/internal/langserver/cmd" ilsp "github.com/hashicorp/terraform-ls/internal/lsp" lsp "github.com/hashicorp/terraform-ls/internal/protocol" + "github.com/hashicorp/terraform-ls/internal/terraform/module" "github.com/hashicorp/terraform-ls/internal/uri" ) @@ -39,19 +40,36 @@ func ModulesHandler(ctx context.Context, args cmd.CommandArgs) (interface{}, err fh := ilsp.FileHandlerFromDocumentURI(lsp.DocumentURI(fileUri)) - cf, err := lsctx.ModuleFinder(ctx) + modMgr, err := lsctx.ModuleManager(ctx) if err != nil { return nil, err } + doneLoading := !walker.IsWalking() - candidates := cf.ModuleCandidatesByPath(fh.Dir()) + + var sources []module.SchemaSource + sources, err = modMgr.SchemaSourcesForModule(fh.Dir()) + if err != nil { + if module.IsModuleNotFound(err) { + _, err := modMgr.AddModule(fh.Dir()) + if err != nil { + return nil, err + } + sources, err = modMgr.SchemaSourcesForModule(fh.Dir()) + if err != nil { + return nil, err + } + } else { + return nil, err + } + } rootDir, _ := lsctx.RootDirectory(ctx) - modules := make([]moduleInfo, len(candidates)) - for i, candidate := range candidates { + modules := make([]moduleInfo, len(sources)) + for i, source := range sources { modules[i] = moduleInfo{ - URI: uri.FromPath(candidate.Path()), - Name: candidate.HumanReadablePath(rootDir), + URI: uri.FromPath(source.Path()), + Name: source.HumanReadablePath(rootDir), } } sort.SliceStable(modules, func(i, j int) bool { diff --git a/internal/langserver/handlers/command/validate.go b/internal/langserver/handlers/command/validate.go index 5d125c0a3..46efe0001 100644 --- a/internal/langserver/handlers/command/validate.go +++ b/internal/langserver/handlers/command/validate.go @@ -7,9 +7,11 @@ import ( "github.com/creachadair/jrpc2/code" lsctx "github.com/hashicorp/terraform-ls/internal/context" "github.com/hashicorp/terraform-ls/internal/langserver/cmd" + "github.com/hashicorp/terraform-ls/internal/langserver/diagnostics" "github.com/hashicorp/terraform-ls/internal/langserver/progress" ilsp "github.com/hashicorp/terraform-ls/internal/lsp" lsp "github.com/hashicorp/terraform-ls/internal/protocol" + "github.com/hashicorp/terraform-ls/internal/terraform/module" ) func TerraformValidateHandler(ctx context.Context, args cmd.CommandArgs) (interface{}, error) { @@ -20,25 +22,29 @@ func TerraformValidateHandler(ctx context.Context, args cmd.CommandArgs) (interf dh := ilsp.FileHandlerFromDirURI(lsp.DocumentURI(dirUri)) - cf, err := lsctx.ModuleFinder(ctx) + modMgr, err := lsctx.ModuleManager(ctx) if err != nil { return nil, err } - mod, err := cf.ModuleByPath(dh.Dir()) + mod, err := modMgr.ModuleByPath(dh.Dir()) if err != nil { - return nil, err + if module.IsModuleNotFound(err) { + mod, err = modMgr.AddModule(dh.Dir()) + if err != nil { + return nil, err + } + } else { + return nil, err + } } - wasInit, err := mod.WasInitialized() + tfExec, err := module.TerraformExecutorForModule(ctx, mod) if err != nil { - return nil, fmt.Errorf("error checking if %s was initialized: %s", dirUri, err) - } - if !wasInit { - return nil, fmt.Errorf("%s is not an initialized module, terraform validate cannot be called", dirUri) + return nil, err } - diags, err := lsctx.Diagnostics(ctx) + notifier, err := lsctx.Diagnostics(ctx) if err != nil { return nil, err } @@ -48,11 +54,14 @@ func TerraformValidateHandler(ctx context.Context, args cmd.CommandArgs) (interf progress.End(ctx, "Finished") }() progress.Report(ctx, "Running terraform validate ...") - hclDiags, err := mod.ExecuteTerraformValidate(ctx) + jsonDiags, err := tfExec.Validate(ctx) if err != nil { return nil, err } - diags.PublishHCLDiags(ctx, mod.Path(), hclDiags, "terraform validate") + + diags := diagnostics.HCLDiagsFromJSON(jsonDiags) + + notifier.PublishHCLDiags(ctx, mod.Path(), diags, "terraform validate") return nil, nil } diff --git a/internal/langserver/handlers/complete.go b/internal/langserver/handlers/complete.go index 091a88069..aaf6c0f1f 100644 --- a/internal/langserver/handlers/complete.go +++ b/internal/langserver/handlers/complete.go @@ -6,6 +6,7 @@ import ( lsctx "github.com/hashicorp/terraform-ls/internal/context" ilsp "github.com/hashicorp/terraform-ls/internal/lsp" lsp "github.com/hashicorp/terraform-ls/internal/protocol" + "github.com/hashicorp/terraform-ls/internal/terraform/module" ) func (h *logHandler) TextDocumentComplete(ctx context.Context, params lsp.CompletionParams) (lsp.CompletionList, error) { @@ -31,20 +32,21 @@ func (h *logHandler) TextDocumentComplete(ctx context.Context, params lsp.Comple return list, err } - module, err := mf.ModuleByPath(file.Dir()) + mod, err := mf.ModuleByPath(file.Dir()) if err != nil { return list, err } - schema, err := mf.SchemaForPath(file.Dir()) + schema, err := mf.SchemaForModule(file.Dir()) if err != nil { return list, err } - d, err := module.DecoderWithSchema(schema) + d, err := module.DecoderForModule(mod) if err != nil { return list, err } + d.SetSchema(schema) fPos, err := ilsp.FilePositionFromDocumentPosition(params.TextDocumentPositionParams, file) if err != nil { diff --git a/internal/langserver/handlers/complete_test.go b/internal/langserver/handlers/complete_test.go index cd80d3be5..fc9557528 100644 --- a/internal/langserver/handlers/complete_test.go +++ b/internal/langserver/handlers/complete_test.go @@ -10,7 +10,6 @@ import ( "github.com/hashicorp/terraform-ls/internal/langserver" "github.com/hashicorp/terraform-ls/internal/langserver/session" "github.com/hashicorp/terraform-ls/internal/terraform/exec" - "github.com/hashicorp/terraform-ls/internal/terraform/module" "github.com/stretchr/testify/mock" ) @@ -43,9 +42,9 @@ func TestCompletion_withValidData(t *testing.T) { } ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: exec.NewMockExecutor([]*mock.Call{ + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): []*mock.Call{ { Method: "Version", Repeatability: 1, @@ -76,7 +75,7 @@ func TestCompletion_withValidData(t *testing.T) { nil, }, }, - }), + }, }, }})) stop := ls.Start(t) @@ -88,7 +87,7 @@ func TestCompletion_withValidData(t *testing.T) { "capabilities": {}, "rootUri": %q, "processId": 12345 - }`, TempDir(t).URI())}) + }`, tmpDir.URI())}) ls.Notify(t, &langserver.CallRequest{ Method: "initialized", ReqParams: "{}", @@ -102,7 +101,7 @@ func TestCompletion_withValidData(t *testing.T) { "text": "provider \"test\" {\n\n}\n", "uri": "%s/main.tf" } - }`, TempDir(t).URI())}) + }`, tmpDir.URI())}) ls.CallAndExpectResponse(t, &langserver.CallRequest{ Method: "textDocument/completion", @@ -114,7 +113,7 @@ func TestCompletion_withValidData(t *testing.T) { "character": 0, "line": 1 } - }`, TempDir(t).URI())}, `{ + }`, tmpDir.URI())}, `{ "jsonrpc": "2.0", "id": 3, "result": { diff --git a/internal/langserver/handlers/did_change.go b/internal/langserver/handlers/did_change.go index 17b50399f..8e74cfc2f 100644 --- a/internal/langserver/handlers/did_change.go +++ b/internal/langserver/handlers/did_change.go @@ -7,6 +7,7 @@ import ( lsctx "github.com/hashicorp/terraform-ls/internal/context" ilsp "github.com/hashicorp/terraform-ls/internal/lsp" lsp "github.com/hashicorp/terraform-ls/internal/protocol" + "github.com/hashicorp/terraform-ls/internal/terraform/module" ) func TextDocumentDidChange(ctx context.Context, params lsp.DidChangeTextDocumentParams) error { @@ -48,17 +49,17 @@ func TextDocumentDidChange(ctx context.Context, params lsp.DidChangeTextDocument return err } - mf, err := lsctx.ModuleFinder(ctx) + modMgr, err := lsctx.ModuleManager(ctx) if err != nil { return err } - module, err := mf.ModuleByPath(fh.Dir()) + mod, err := modMgr.ModuleByPath(fh.Dir()) if err != nil { return err } - err = module.ParseFiles() + err = modMgr.EnqueueModuleOpWait(mod.Path(), module.OpTypeParseConfiguration) if err != nil { return err } @@ -67,7 +68,7 @@ func TextDocumentDidChange(ctx context.Context, params lsp.DidChangeTextDocument if err != nil { return err } - diags.PublishHCLDiags(ctx, module.Path(), module.ParsedDiagnostics(), "HCL") + diags.PublishHCLDiags(ctx, mod.Path(), mod.Diagnostics(), "HCL") return nil } diff --git a/internal/langserver/handlers/did_change_test.go b/internal/langserver/handlers/did_change_test.go index 62f9df908..d1316095d 100644 --- a/internal/langserver/handlers/did_change_test.go +++ b/internal/langserver/handlers/did_change_test.go @@ -9,7 +9,8 @@ import ( "github.com/hashicorp/terraform-ls/internal/filesystem" "github.com/hashicorp/terraform-ls/internal/langserver" "github.com/hashicorp/terraform-ls/internal/lsp" - "github.com/hashicorp/terraform-ls/internal/terraform/module" + "github.com/hashicorp/terraform-ls/internal/terraform/exec" + "github.com/stretchr/testify/mock" ) func TestLangServer_didChange_sequenceOfPartialChanges(t *testing.T) { @@ -18,9 +19,9 @@ func TestLangServer_didChange_sequenceOfPartialChanges(t *testing.T) { fs := filesystem.NewFilesystem() ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: validTfMockCalls(), + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), }, }, Filesystem: fs, diff --git a/internal/langserver/handlers/did_open.go b/internal/langserver/handlers/did_open.go index b55b4a016..8c89c6142 100644 --- a/internal/langserver/handlers/did_open.go +++ b/internal/langserver/handlers/did_open.go @@ -46,7 +46,7 @@ func (lh *logHandler) TextDocumentDidOpen(ctx context.Context, params lsp.DidOpe mod, err = modMgr.ModuleByPath(f.Dir()) if err != nil { if module.IsModuleNotFound(err) { - mod, err = modMgr.AddAndStartLoadingModule(ctx, f.Dir()) + mod, err = modMgr.AddModule(f.Dir()) if err != nil { return err } @@ -60,23 +60,39 @@ func (lh *logHandler) TextDocumentDidOpen(ctx context.Context, params lsp.DidOpe // We reparse because the file being opened may not match // (originally parsed) content on the disk // TODO: Do this only if we can verify the file differs? - err = mod.ParseFiles() + modMgr.EnqueueModuleOpWait(mod.Path(), module.OpTypeParseConfiguration) + + if mod.TerraformVersionState() == module.OpStateUnknown { + modMgr.EnqueueModuleOp(mod.Path(), module.OpTypeGetTerraformVersion) + } + + watcher, err := lsctx.Watcher(ctx) if err != nil { - return fmt.Errorf("failed to parse files: %w", err) + return err + } + + if !watcher.IsModuleWatched(mod.Path()) { + err := watcher.AddModule(mod.Path()) + if err != nil { + return err + } } diags, err := lsctx.Diagnostics(ctx) if err != nil { return err } - diags.PublishHCLDiags(ctx, mod.Path(), mod.ParsedDiagnostics(), "HCL") + diags.PublishHCLDiags(ctx, mod.Path(), mod.Diagnostics(), "HCL") - candidates := modMgr.ModuleCandidatesByPath(f.Dir()) + sources, err := modMgr.SchemaSourcesForModule(f.Dir()) + if err != nil { + return err + } - if walker.IsWalking() { - // avoid raising false warnings if walker hasn't finished yet + if walker.IsWalking() || mod.ProviderSchemaState() == module.OpStateLoading { + // avoid raising false warnings if operations are still in-flight lh.logger.Printf("walker has not finished walking yet, data may be inaccurate for %s", f.FullPath()) - } else if len(candidates) == 0 { + } else if len(sources) == 0 { // TODO: Only notify once per f.Dir() per session dh := ilsp.FileHandlerFromDirPath(f.Dir()) go func() { @@ -90,12 +106,12 @@ func (lh *logHandler) TextDocumentDidOpen(ctx context.Context, params lsp.DidOpe }() } - if len(candidates) > 1 { - candidateDir := humanReadablePath(rootDir, candidates[0].Path()) + if len(sources) > 1 { + candidateDir := humanReadablePath(rootDir, sources[0].Path()) msg := fmt.Sprintf("Alternative schema source found for %s (%s), picked: %s."+ " You can set an explicit module path in your settings.", - readableDir, candidatePaths(rootDir, candidates[1:]), + readableDir, candidatePaths(rootDir, sources[1:]), candidateDir) return jrpc2.PushNotify(ctx, "window/showMessage", lsp.ShowMessageParams{ Type: lsp.Warning, @@ -106,10 +122,10 @@ func (lh *logHandler) TextDocumentDidOpen(ctx context.Context, params lsp.DidOpe return nil } -func candidatePaths(rootDir string, candidates []module.Module) string { - paths := make([]string, len(candidates)) - for i, mod := range candidates { - paths[i] = humanReadablePath(rootDir, mod.Path()) +func candidatePaths(rootDir string, sources []module.SchemaSource) string { + paths := make([]string, len(sources)) + for i, source := range sources { + paths[i] = humanReadablePath(rootDir, source.Path()) } return strings.Join(paths, ", ") } diff --git a/internal/langserver/handlers/execute_command_init_test.go b/internal/langserver/handlers/execute_command_init_test.go index 0def28f5e..ec4074bb9 100644 --- a/internal/langserver/handlers/execute_command_init_test.go +++ b/internal/langserver/handlers/execute_command_init_test.go @@ -10,7 +10,6 @@ import ( "github.com/hashicorp/terraform-ls/internal/langserver" "github.com/hashicorp/terraform-ls/internal/langserver/cmd" "github.com/hashicorp/terraform-ls/internal/terraform/exec" - "github.com/hashicorp/terraform-ls/internal/terraform/module" "github.com/stretchr/testify/mock" ) @@ -19,9 +18,9 @@ func TestLangServer_workspaceExecuteCommand_init_argumentError(t *testing.T) { testFileURI := fmt.Sprintf("%s/main.tf", tmpDir.URI()) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: validTfMockCalls(), + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), }, }, })) @@ -61,7 +60,7 @@ func TestLangServer_workspaceExecuteCommand_init_basic(t *testing.T) { tmpDir := TempDir(t) testFileURI := fmt.Sprintf("%s/main.tf", tmpDir.URI()) - tfMockCalls := exec.NewMockExecutor([]*mock.Call{ + tfMockCalls := []*mock.Call{ { Method: "Version", Repeatability: 1, @@ -91,12 +90,12 @@ func TestLangServer_workspaceExecuteCommand_init_basic(t *testing.T) { nil, }, }, - }) + } ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: tfMockCalls, + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): tfMockCalls, }, }, })) @@ -141,7 +140,7 @@ func TestLangServer_workspaceExecuteCommand_init_error(t *testing.T) { tmpDir := TempDir(t) testFileURI := fmt.Sprintf("%s/main.tf", tmpDir.URI()) - tfMockCalls := exec.NewMockExecutor([]*mock.Call{ + tfMockCalls := []*mock.Call{ { Method: "Version", Repeatability: 1, @@ -171,12 +170,12 @@ func TestLangServer_workspaceExecuteCommand_init_error(t *testing.T) { errors.New("something bad happened"), }, }, - }) + } ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: tfMockCalls, + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): tfMockCalls, }, }, })) diff --git a/internal/langserver/handlers/execute_command_modules_test.go b/internal/langserver/handlers/execute_command_modules_test.go index 7582ceac5..e700797de 100644 --- a/internal/langserver/handlers/execute_command_modules_test.go +++ b/internal/langserver/handlers/execute_command_modules_test.go @@ -9,7 +9,8 @@ import ( "github.com/hashicorp/terraform-ls/internal/langserver" "github.com/hashicorp/terraform-ls/internal/langserver/cmd" "github.com/hashicorp/terraform-ls/internal/lsp" - "github.com/hashicorp/terraform-ls/internal/terraform/module" + "github.com/hashicorp/terraform-ls/internal/terraform/exec" + "github.com/stretchr/testify/mock" ) func TestLangServer_workspaceExecuteCommand_modules_argumentError(t *testing.T) { @@ -18,9 +19,9 @@ func TestLangServer_workspaceExecuteCommand_modules_argumentError(t *testing.T) InitPluginCache(t, tmpDir.Dir()) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: validTfMockCalls(), + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), }, }, })) @@ -62,9 +63,9 @@ func TestLangServer_workspaceExecuteCommand_modules_basic(t *testing.T) { InitPluginCache(t, tmpDir.Dir()) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: validTfMockCalls(), + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), }, }, })) @@ -128,15 +129,11 @@ func TestLangServer_workspaceExecuteCommand_modules_multiple(t *testing.T) { prod := lsp.FileHandlerFromDirPath(filepath.Join(testData, "main-module-multienv", "env", "prod")) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - dev.Dir(): { - TfExecFactory: validTfMockCalls(), - }, - staging.Dir(): { - TfExecFactory: validTfMockCalls(), - }, - prod.Dir(): { - TfExecFactory: validTfMockCalls(), + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + dev.Dir(): validTfMockCalls(), + staging.Dir(): validTfMockCalls(), + prod.Dir(): validTfMockCalls(), }, }, })) diff --git a/internal/langserver/handlers/execute_command_test.go b/internal/langserver/handlers/execute_command_test.go index 2bb52bf9e..7d7d24be2 100644 --- a/internal/langserver/handlers/execute_command_test.go +++ b/internal/langserver/handlers/execute_command_test.go @@ -6,7 +6,8 @@ import ( "github.com/creachadair/jrpc2/code" "github.com/hashicorp/terraform-ls/internal/langserver" - "github.com/hashicorp/terraform-ls/internal/terraform/module" + "github.com/hashicorp/terraform-ls/internal/terraform/exec" + "github.com/stretchr/testify/mock" ) func TestLangServer_workspaceExecuteCommand_noCommandHandlerError(t *testing.T) { @@ -16,9 +17,9 @@ func TestLangServer_workspaceExecuteCommand_noCommandHandlerError(t *testing.T) InitPluginCache(t, tmpDir.Dir()) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: validTfMockCalls(), + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), }, }, })) diff --git a/internal/langserver/handlers/execute_command_validate_test.go b/internal/langserver/handlers/execute_command_validate_test.go index 95d313fb0..c6cfeaebb 100644 --- a/internal/langserver/handlers/execute_command_validate_test.go +++ b/internal/langserver/handlers/execute_command_validate_test.go @@ -7,7 +7,8 @@ import ( "github.com/creachadair/jrpc2/code" "github.com/hashicorp/terraform-ls/internal/langserver" "github.com/hashicorp/terraform-ls/internal/langserver/cmd" - "github.com/hashicorp/terraform-ls/internal/terraform/module" + "github.com/hashicorp/terraform-ls/internal/terraform/exec" + "github.com/stretchr/testify/mock" ) func TestLangServer_workspaceExecuteCommand_validate_argumentError(t *testing.T) { @@ -15,9 +16,9 @@ func TestLangServer_workspaceExecuteCommand_validate_argumentError(t *testing.T) testFileURI := fmt.Sprintf("%s/main.tf", tmpDir.URI()) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: validTfMockCalls(), + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), }, }, })) diff --git a/internal/langserver/handlers/formatting.go b/internal/langserver/handlers/formatting.go index e52c0303c..40032501d 100644 --- a/internal/langserver/handlers/formatting.go +++ b/internal/langserver/handlers/formatting.go @@ -2,13 +2,11 @@ package handlers import ( "context" - "fmt" lsctx "github.com/hashicorp/terraform-ls/internal/context" "github.com/hashicorp/terraform-ls/internal/hcl" ilsp "github.com/hashicorp/terraform-ls/internal/lsp" lsp "github.com/hashicorp/terraform-ls/internal/protocol" - "github.com/hashicorp/terraform-ls/internal/terraform/exec" "github.com/hashicorp/terraform-ls/internal/terraform/module" ) @@ -20,18 +18,24 @@ func (h *logHandler) TextDocumentFormatting(ctx context.Context, params lsp.Docu return edits, err } - tff, err := lsctx.TerraformFormatterFinder(ctx) + mf, err := lsctx.ModuleFinder(ctx) if err != nil { return edits, err } fh := ilsp.FileHandlerFromDocumentURI(params.TextDocument.URI) - file, err := fs.GetDocument(fh) + + mod, err := mf.ModuleByPath(fh.Dir()) + if err != nil { + return edits, err + } + + tfExec, err := module.TerraformExecutorForModule(ctx, mod) if err != nil { return edits, err } - format, err := findTerraformFormatter(ctx, tff, file.Dir()) + file, err := fs.GetDocument(fh) if err != nil { return edits, err } @@ -41,7 +45,7 @@ func (h *logHandler) TextDocumentFormatting(ctx context.Context, params lsp.Docu return edits, err } - formatted, err := format(ctx, original) + formatted, err := tfExec.Format(ctx, original) if err != nil { return edits, err } @@ -50,30 +54,3 @@ func (h *logHandler) TextDocumentFormatting(ctx context.Context, params lsp.Docu return ilsp.TextEditsFromDocumentChanges(changes), nil } - -func findTerraformFormatter(ctx context.Context, tff module.TerraformFormatterFinder, dir string) (exec.Formatter, error) { - discoveryDone, err := tff.HasTerraformDiscoveryFinished(dir) - if err != nil { - if module.IsModuleNotFound(err) { - return tff.TerraformFormatterForDir(ctx, dir) - } - return nil, err - } else { - if !discoveryDone { - // TODO: block until it's available <-tff.TerraformLoadingDone() - return nil, fmt.Errorf("terraform is still being discovered for %s", dir) - } - available, err := tff.IsTerraformAvailable(dir) - if err != nil { - if module.IsModuleNotFound(err) { - return tff.TerraformFormatterForDir(ctx, dir) - } - } - if !available { - // TODO: block until it's available <-tff.TerraformLoadingDone() - return nil, fmt.Errorf("terraform is not available for %s", dir) - } - } - - return tff.TerraformFormatterForDir(ctx, dir) -} diff --git a/internal/langserver/handlers/formatting_test.go b/internal/langserver/handlers/formatting_test.go index 923597d3d..8c407a027 100644 --- a/internal/langserver/handlers/formatting_test.go +++ b/internal/langserver/handlers/formatting_test.go @@ -10,7 +10,6 @@ import ( "github.com/hashicorp/terraform-ls/internal/langserver" "github.com/hashicorp/terraform-ls/internal/langserver/session" "github.com/hashicorp/terraform-ls/internal/terraform/exec" - "github.com/hashicorp/terraform-ls/internal/terraform/module" "github.com/stretchr/testify/mock" ) @@ -35,9 +34,9 @@ func TestLangServer_formatting_basic(t *testing.T) { tmpDir := TempDir(t) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: exec.NewMockExecutor([]*mock.Call{ + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): { { Method: "Version", Repeatability: 1, @@ -69,7 +68,7 @@ func TestLangServer_formatting_basic(t *testing.T) { nil, }, }, - }), + }, }, }, })) @@ -121,9 +120,9 @@ func TestLangServer_formatting_basic(t *testing.T) { func TestLangServer_formatting_oldVersion(t *testing.T) { tmpDir := TempDir(t) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: exec.NewMockExecutor([]*mock.Call{ + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): { { Method: "Version", Repeatability: 1, @@ -155,7 +154,7 @@ func TestLangServer_formatting_oldVersion(t *testing.T) { errors.New("not implemented"), }, }, - }), + }, }, }, })) diff --git a/internal/langserver/handlers/handlers_test.go b/internal/langserver/handlers/handlers_test.go index 0e981873d..e97ca636d 100644 --- a/internal/langserver/handlers/handlers_test.go +++ b/internal/langserver/handlers/handlers_test.go @@ -13,7 +13,6 @@ import ( "github.com/hashicorp/terraform-ls/internal/langserver" "github.com/hashicorp/terraform-ls/internal/lsp" "github.com/hashicorp/terraform-ls/internal/terraform/exec" - "github.com/hashicorp/terraform-ls/internal/terraform/module" "github.com/stretchr/testify/mock" ) @@ -69,9 +68,12 @@ func TestInitalizeAndShutdown(t *testing.T) { tmpDir := TempDir(t) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): {TfExecFactory: validTfMockCalls()}, - }})) + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), + }, + }, + })) stop := ls.Start(t) defer stop() @@ -95,9 +97,12 @@ func TestInitalizeWithCommandPrefix(t *testing.T) { tmpDir := TempDir(t) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): {TfExecFactory: validTfMockCalls()}, - }})) + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), + }, + }, + })) stop := ls.Start(t) defer stop() @@ -117,9 +122,12 @@ func TestEOF(t *testing.T) { tmpDir := TempDir(t) ms := newMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - TempDir(t).Dir(): {TfExecFactory: validTfMockCalls()}, - }}) + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), + }, + }, + }) ls := langserver.NewLangServerMock(t, ms.new) stop := ls.Start(t) defer stop() @@ -146,8 +154,8 @@ func TestEOF(t *testing.T) { } } -func validTfMockCalls() exec.ExecutorFactory { - return exec.NewMockExecutor([]*mock.Call{ +func validTfMockCalls() []*mock.Call { + return []*mock.Call{ { Method: "Version", Repeatability: 1, @@ -178,7 +186,7 @@ func validTfMockCalls() exec.ExecutorFactory { nil, }, }, - }) + } } // TempDir creates a temporary directory containing the test name, as well any diff --git a/internal/langserver/handlers/hover.go b/internal/langserver/handlers/hover.go index cf588a215..b654e3348 100644 --- a/internal/langserver/handlers/hover.go +++ b/internal/langserver/handlers/hover.go @@ -6,6 +6,7 @@ import ( lsctx "github.com/hashicorp/terraform-ls/internal/context" ilsp "github.com/hashicorp/terraform-ls/internal/lsp" lsp "github.com/hashicorp/terraform-ls/internal/protocol" + "github.com/hashicorp/terraform-ls/internal/terraform/module" ) func (h *logHandler) TextDocumentHover(ctx context.Context, params lsp.TextDocumentPositionParams) (*lsp.Hover, error) { @@ -34,15 +35,16 @@ func (h *logHandler) TextDocumentHover(ctx context.Context, params lsp.TextDocum return nil, err } - schema, err := mf.SchemaForPath(file.Dir()) + schema, err := mf.SchemaForModule(file.Dir()) if err != nil { return nil, err } - d, err := mod.DecoderWithSchema(schema) + d, err := module.DecoderForModule(mod) if err != nil { return nil, err } + d.SetSchema(schema) fPos, err := ilsp.FilePositionFromDocumentPosition(params, file) if err != nil { diff --git a/internal/langserver/handlers/hover_test.go b/internal/langserver/handlers/hover_test.go index 74812864e..de57ee3b6 100644 --- a/internal/langserver/handlers/hover_test.go +++ b/internal/langserver/handlers/hover_test.go @@ -10,7 +10,6 @@ import ( "github.com/hashicorp/terraform-ls/internal/langserver" "github.com/hashicorp/terraform-ls/internal/langserver/session" "github.com/hashicorp/terraform-ls/internal/terraform/exec" - "github.com/hashicorp/terraform-ls/internal/terraform/module" "github.com/stretchr/testify/mock" ) @@ -43,9 +42,9 @@ func TestHover_withValidData(t *testing.T) { } ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: exec.NewMockExecutor([]*mock.Call{ + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): { { Method: "Version", Repeatability: 1, @@ -76,9 +75,10 @@ func TestHover_withValidData(t *testing.T) { nil, }, }, - }), + }, }, - }})) + }, + })) stop := ls.Start(t) defer stop() diff --git a/internal/langserver/handlers/initialize.go b/internal/langserver/handlers/initialize.go index e4e5281d8..d21709f2f 100644 --- a/internal/langserver/handlers/initialize.go +++ b/internal/langserver/handlers/initialize.go @@ -57,16 +57,6 @@ func (lh *logHandler) Initialize(ctx context.Context, params lsp.InitializeParam return serverCaps, err } - modMgr, err := lsctx.ModuleManager(ctx) - if err != nil { - return serverCaps, err - } - - addAndLoadModule, err := lsctx.ModuleLoader(ctx) - if err != nil { - return serverCaps, err - } - w, err := lsctx.Watcher(ctx) if err != nil { return serverCaps, err @@ -128,14 +118,8 @@ func (lh *logHandler) Initialize(ctx context.Context, params lsp.InitializeParam }) continue } - mod, err := addAndLoadModule(modPath) - if err != nil { - return serverCaps, err - } - paths := mod.PathsToWatch() - lh.logger.Printf("Adding %d module paths for watching (%s)", len(paths), modPath) - err = w.AddPaths(paths) + err = w.AddModule(modPath) if err != nil { return serverCaps, err } @@ -164,22 +148,7 @@ func (lh *logHandler) Initialize(ctx context.Context, params lsp.InitializeParam // Walker runs asynchronously so we're intentionally *not* // passing the request context here bCtx := context.Background() - err = walker.StartWalking(bCtx, fh.Dir(), func(ctx context.Context, dir string) error { - lh.logger.Printf("Adding module: %s", dir) - mod, err := modMgr.AddAndStartLoadingModule(ctx, dir) - if err != nil { - return err - } - - paths := mod.PathsToWatch() - lh.logger.Printf("Adding %d paths of module for watching (%s)", len(paths), dir) - err = w.AddPaths(paths) - if err != nil { - return err - } - - return nil - }) + err = walker.StartWalking(bCtx, fh.Dir()) return serverCaps, err } diff --git a/internal/langserver/handlers/initialize_test.go b/internal/langserver/handlers/initialize_test.go index edefbc543..3ba93ceb6 100644 --- a/internal/langserver/handlers/initialize_test.go +++ b/internal/langserver/handlers/initialize_test.go @@ -8,15 +8,18 @@ import ( "github.com/hashicorp/go-version" "github.com/hashicorp/terraform-ls/internal/langserver" "github.com/hashicorp/terraform-ls/internal/terraform/exec" - "github.com/hashicorp/terraform-ls/internal/terraform/module" "github.com/stretchr/testify/mock" ) func TestInitialize_twice(t *testing.T) { + tmpDir := TempDir(t) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - TempDir(t).Dir(): {TfExecFactory: validTfMockCalls()}, - }})) + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), + }, + }, + })) stop := ls.Start(t) defer stop() @@ -39,9 +42,9 @@ func TestInitialize_twice(t *testing.T) { func TestInitialize_withIncompatibleTerraformVersion(t *testing.T) { tmpDir := TempDir(t) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: exec.NewMockExecutor([]*mock.Call{ + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): { { Method: "Version", Repeatability: 1, @@ -53,9 +56,10 @@ func TestInitialize_withIncompatibleTerraformVersion(t *testing.T) { nil, }, }, - }), + }, }, - }})) + }, + })) stop := ls.Start(t) defer stop() @@ -69,10 +73,14 @@ func TestInitialize_withIncompatibleTerraformVersion(t *testing.T) { } func TestInitialize_withInvalidRootURI(t *testing.T) { + tmpDir := TempDir(t) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - TempDir(t).Dir(): {TfExecFactory: validTfMockCalls()}, - }})) + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), + }, + }, + })) stop := ls.Start(t) defer stop() diff --git a/internal/langserver/handlers/semantic_tokens.go b/internal/langserver/handlers/semantic_tokens.go index 62cefab46..9ad706d0b 100644 --- a/internal/langserver/handlers/semantic_tokens.go +++ b/internal/langserver/handlers/semantic_tokens.go @@ -8,6 +8,7 @@ import ( lsctx "github.com/hashicorp/terraform-ls/internal/context" ilsp "github.com/hashicorp/terraform-ls/internal/lsp" lsp "github.com/hashicorp/terraform-ls/internal/protocol" + "github.com/hashicorp/terraform-ls/internal/terraform/module" ) func (lh *logHandler) TextDocumentSemanticTokensFull(ctx context.Context, params lsp.SemanticTokensParams) (lsp.SemanticTokens, error) { @@ -50,15 +51,16 @@ func (lh *logHandler) TextDocumentSemanticTokensFull(ctx context.Context, params return tks, fmt.Errorf("finding compatible decoder failed: %w", err) } - schema, err := mf.SchemaForPath(doc.Dir()) + schema, err := mf.SchemaForModule(doc.Dir()) if err != nil { return tks, err } - d, err := mod.DecoderWithSchema(schema) + d, err := module.DecoderForModule(mod) if err != nil { return tks, err } + d.SetSchema(schema) tokens, err := d.SemanticTokensInFile(doc.Filename()) if err != nil { diff --git a/internal/langserver/handlers/semantic_tokens_test.go b/internal/langserver/handlers/semantic_tokens_test.go index ae971785d..ff0633a76 100644 --- a/internal/langserver/handlers/semantic_tokens_test.go +++ b/internal/langserver/handlers/semantic_tokens_test.go @@ -9,7 +9,6 @@ import ( tfjson "github.com/hashicorp/terraform-json" "github.com/hashicorp/terraform-ls/internal/langserver" "github.com/hashicorp/terraform-ls/internal/terraform/exec" - "github.com/hashicorp/terraform-ls/internal/terraform/module" "github.com/stretchr/testify/mock" ) @@ -24,9 +23,9 @@ func TestSemanticTokensFull(t *testing.T) { } ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: exec.NewMockExecutor([]*mock.Call{ + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): { { Method: "Version", Repeatability: 1, @@ -57,7 +56,7 @@ func TestSemanticTokensFull(t *testing.T) { nil, }, }, - }), + }, }, }})) stop := ls.Start(t) @@ -131,9 +130,9 @@ func TestSemanticTokensFull_clientSupportsDelta(t *testing.T) { } ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: exec.NewMockExecutor([]*mock.Call{ + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): { { Method: "Version", Repeatability: 1, @@ -164,7 +163,7 @@ func TestSemanticTokensFull_clientSupportsDelta(t *testing.T) { nil, }, }, - }), + }, }, }})) stop := ls.Start(t) diff --git a/internal/langserver/handlers/service.go b/internal/langserver/handlers/service.go index 90da02b04..c19150c05 100644 --- a/internal/langserver/handlers/service.go +++ b/internal/langserver/handlers/service.go @@ -6,7 +6,6 @@ import ( "fmt" "io/ioutil" "log" - "time" "github.com/creachadair/jrpc2" "github.com/creachadair/jrpc2/code" @@ -17,8 +16,9 @@ import ( "github.com/hashicorp/terraform-ls/internal/langserver/session" lsp "github.com/hashicorp/terraform-ls/internal/protocol" "github.com/hashicorp/terraform-ls/internal/settings" + "github.com/hashicorp/terraform-ls/internal/terraform/discovery" + "github.com/hashicorp/terraform-ls/internal/terraform/exec" "github.com/hashicorp/terraform-ls/internal/terraform/module" - "github.com/hashicorp/terraform-ls/internal/watcher" ) type service struct { @@ -30,18 +30,21 @@ type service struct { stopSession context.CancelFunc fs filesystem.Filesystem - watcher watcher.Watcher + watcher module.Watcher walker *module.Walker modMgr module.ModuleManager newModuleManager module.ModuleManagerFactory - newWatcher watcher.WatcherFactory + newWatcher module.WatcherFactory newWalker module.WalkerFactory + tfDiscoFunc discovery.DiscoveryFunc + tfExecFactory exec.ExecutorFactory } var discardLogs = log.New(ioutil.Discard, "", 0) func NewSession(srvCtx context.Context) session.Session { fs := filesystem.NewFilesystem() + d := &discovery.Discovery{} sessCtx, stopSession := context.WithCancel(srvCtx) return &service{ @@ -51,8 +54,10 @@ func NewSession(srvCtx context.Context) session.Session { sessCtx: sessCtx, stopSession: stopSession, newModuleManager: module.NewModuleManager, - newWatcher: watcher.NewWatcher, + newWatcher: module.NewWatcher, newWalker: module.NewWalker, + tfDiscoFunc: d.LookPath, + tfExecFactory: exec.NewExecutor, } } @@ -77,76 +82,42 @@ func (svc *service) Assigner() (jrpc2.Assigner, error) { lh := LogHandler(svc.logger) cc := &lsp.ClientCapabilities{} - svc.modMgr = svc.newModuleManager(svc.fs) - svc.modMgr.SetLogger(svc.logger) - - svc.logger.Printf("Worker pool size set to %d", svc.modMgr.WorkerPoolSize()) - - tr := newTickReporter(5 * time.Second) - tr.AddReporter(func() { - queueSize := svc.modMgr.WorkerQueueSize() - if queueSize < 1 { - return - } - svc.logger.Printf("Root modules waiting to be loaded: %d", queueSize) - }) - tr.StartReporting(svc.sessCtx) - - svc.walker = svc.newWalker() - // The following is set via CLI flags, hence available in the server context + execOpts := &exec.ExecutorOpts{} if path, ok := lsctx.TerraformExecPath(svc.srvCtx); ok { - svc.modMgr.SetTerraformExecPath(path) + execOpts.ExecPath = path + } else { + tfExecPath, err := svc.tfDiscoFunc() + if err == nil { + execOpts.ExecPath = tfExecPath + } } if path, ok := lsctx.TerraformExecLogPath(svc.srvCtx); ok { - svc.modMgr.SetTerraformExecLogPath(path) + execOpts.ExecLogPath = path } if timeout, ok := lsctx.TerraformExecTimeout(svc.srvCtx); ok { - svc.modMgr.SetTerraformExecTimeout(timeout) + execOpts.Timeout = timeout } - ww, err := svc.newWatcher() + svc.sessCtx = exec.WithExecutorOpts(svc.sessCtx, execOpts) + svc.sessCtx = exec.WithExecutorFactory(svc.sessCtx, svc.tfExecFactory) + + svc.modMgr = svc.newModuleManager(svc.sessCtx, svc.fs) + svc.modMgr.SetLogger(svc.logger) + + svc.walker = svc.newWalker(svc.fs, svc.modMgr) + + ww, err := svc.newWatcher(svc.fs, svc.modMgr) if err != nil { return nil, err } svc.watcher = ww svc.watcher.SetLogger(svc.logger) - svc.watcher.AddChangeHook(func(ctx context.Context, file watcher.TrackedFile) error { - mod, err := svc.modMgr.ModuleByPath(file.Path()) - if err != nil { - return err - } - if mod.IsKnownPluginLockFile(file.Path()) { - svc.logger.Printf("detected plugin cache change, updating schema ...") - err := mod.UpdateProviderSchemaCache(ctx, file) - if err != nil { - svc.logger.Printf(err.Error()) - } - } - - return nil - }) - svc.watcher.AddChangeHook(func(_ context.Context, file watcher.TrackedFile) error { - mod, err := svc.modMgr.ModuleByPath(file.Path()) - if err != nil { - return err - } - if mod.IsKnownModuleManifestFile(file.Path()) { - svc.logger.Printf("detected module manifest change, updating ...") - err := mod.UpdateModuleManifest(file) - if err != nil { - svc.logger.Printf(err.Error()) - } - } - - return nil - }) err = svc.watcher.Start() if err != nil { return nil, err } - modLoader := module.NewModuleLoader(svc.sessCtx, svc.modMgr) diags := diagnostics.NewNotifier(svc.sessCtx, svc.logger) rootDir := "" @@ -166,7 +137,6 @@ func (svc *service) Assigner() (jrpc2.Assigner, error) { ctx = lsctx.WithRootDirectory(ctx, &rootDir) ctx = lsctx.WithCommandPrefix(ctx, &commandPrefix) ctx = lsctx.WithModuleManager(ctx, svc.modMgr) - ctx = lsctx.WithModuleLoader(ctx, modLoader) ctx = lsctx.WithExperimentalFeatures(ctx, &expFeatures) version, ok := lsctx.LanguageServerVersion(svc.srvCtx) @@ -191,7 +161,7 @@ func (svc *service) Assigner() (jrpc2.Assigner, error) { } ctx = lsctx.WithDiagnostics(ctx, diags) ctx = lsctx.WithDocumentStorage(ctx, svc.fs) - ctx = lsctx.WithModuleFinder(ctx, svc.modMgr) + ctx = lsctx.WithModuleManager(ctx, svc.modMgr) return handle(ctx, req, TextDocumentDidChange) }, "textDocument/didOpen": func(ctx context.Context, req *jrpc2.Request) (interface{}, error) { @@ -259,7 +229,9 @@ func (svc *service) Assigner() (jrpc2.Assigner, error) { } ctx = lsctx.WithDocumentStorage(ctx, svc.fs) - ctx = lsctx.WithTerraformFormatterFinder(ctx, svc.modMgr) + ctx = lsctx.WithModuleFinder(ctx, svc.modMgr) + ctx = exec.WithExecutorOpts(ctx, execOpts) + ctx = exec.WithExecutorFactory(ctx, svc.tfExecFactory) return handle(ctx, req, lh.TextDocumentFormatting) }, @@ -284,6 +256,7 @@ func (svc *service) Assigner() (jrpc2.Assigner, error) { ctx = lsctx.WithDiagnostics(ctx, diags) ctx = lsctx.WithExperimentalFeatures(ctx, &expFeatures) ctx = lsctx.WithModuleFinder(ctx, svc.modMgr) + ctx = exec.WithExecutorOpts(ctx, execOpts) return handle(ctx, req, lh.TextDocumentDidSave) }, @@ -294,11 +267,13 @@ func (svc *service) Assigner() (jrpc2.Assigner, error) { } ctx = lsctx.WithCommandPrefix(ctx, &commandPrefix) - ctx = lsctx.WithModuleFinder(ctx, svc.modMgr) + ctx = lsctx.WithModuleManager(ctx, svc.modMgr) ctx = lsctx.WithModuleWalker(ctx, svc.walker) ctx = lsctx.WithWatcher(ctx, ww) ctx = lsctx.WithRootDirectory(ctx, &rootDir) ctx = lsctx.WithDiagnostics(ctx, diags) + ctx = exec.WithExecutorOpts(ctx, execOpts) + ctx = exec.WithExecutorFactory(ctx, svc.tfExecFactory) return handle(ctx, req, lh.WorkspaceExecuteCommand) }, diff --git a/internal/langserver/handlers/service_mock_test.go b/internal/langserver/handlers/session_mock_test.go similarity index 75% rename from internal/langserver/handlers/service_mock_test.go rename to internal/langserver/handlers/session_mock_test.go index cb570a6a9..345aaf9a6 100644 --- a/internal/langserver/handlers/service_mock_test.go +++ b/internal/langserver/handlers/session_mock_test.go @@ -9,15 +9,14 @@ import ( "github.com/hashicorp/terraform-ls/internal/filesystem" "github.com/hashicorp/terraform-ls/internal/langserver/session" + "github.com/hashicorp/terraform-ls/internal/terraform/discovery" "github.com/hashicorp/terraform-ls/internal/terraform/exec" "github.com/hashicorp/terraform-ls/internal/terraform/module" - "github.com/hashicorp/terraform-ls/internal/watcher" ) type MockSessionInput struct { - Modules map[string]*module.ModuleMock - Filesystem filesystem.Filesystem - TfExecutorFactory exec.ExecutorFactory + Filesystem filesystem.Filesystem + TerraformCalls *exec.TerraformMockCalls } type mockSession struct { @@ -34,8 +33,7 @@ func (ms *mockSession) new(srvCtx context.Context) session.Session { var input *module.ModuleManagerMockInput if ms.mockInput != nil { input = &module.ModuleManagerMockInput{ - Modules: ms.mockInput.Modules, - TfExecutorFactory: ms.mockInput.TfExecutorFactory, + Logger: testLogger(), } } @@ -46,6 +44,15 @@ func (ms *mockSession) new(srvCtx context.Context) session.Session { fs = filesystem.NewFilesystem() } + var tfCalls *exec.TerraformMockCalls + if ms.mockInput != nil && ms.mockInput.TerraformCalls != nil { + tfCalls = ms.mockInput.TerraformCalls + } + + d := &discovery.MockDiscovery{ + Path: "tf-mock", + } + svc := &service{ logger: testLogger(), srvCtx: srvCtx, @@ -53,8 +60,10 @@ func (ms *mockSession) new(srvCtx context.Context) session.Session { stopSession: ms.stop, fs: fs, newModuleManager: module.NewModuleManagerMock(input), - newWatcher: watcher.MockWatcher(), - newWalker: module.MockWalker, + newWatcher: module.MockWatcher(), + newWalker: module.SyncWalker, + tfDiscoFunc: d.LookPath, + tfExecFactory: exec.NewMockExecutor(tfCalls), } return svc diff --git a/internal/langserver/handlers/shutdown_test.go b/internal/langserver/handlers/shutdown_test.go index c99541d0c..959480905 100644 --- a/internal/langserver/handlers/shutdown_test.go +++ b/internal/langserver/handlers/shutdown_test.go @@ -6,14 +6,19 @@ import ( "github.com/creachadair/jrpc2/code" "github.com/hashicorp/terraform-ls/internal/langserver" - "github.com/hashicorp/terraform-ls/internal/terraform/module" + "github.com/hashicorp/terraform-ls/internal/terraform/exec" + "github.com/stretchr/testify/mock" ) func TestShutdown_twice(t *testing.T) { + tmpDir := TempDir(t) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - TempDir(t).Dir(): {TfExecFactory: validTfMockCalls()}, - }})) + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), + }, + }, + })) stop := ls.Start(t) defer stop() diff --git a/internal/langserver/handlers/symbols.go b/internal/langserver/handlers/symbols.go index fa8c8daa0..1bf99a3f2 100644 --- a/internal/langserver/handlers/symbols.go +++ b/internal/langserver/handlers/symbols.go @@ -2,11 +2,11 @@ package handlers import ( "context" - "fmt" lsctx "github.com/hashicorp/terraform-ls/internal/context" ilsp "github.com/hashicorp/terraform-ls/internal/lsp" lsp "github.com/hashicorp/terraform-ls/internal/protocol" + "github.com/hashicorp/terraform-ls/internal/terraform/module" ) func (h *logHandler) TextDocumentSymbol(ctx context.Context, params lsp.DocumentSymbolParams) ([]lsp.SymbolInformation, error) { @@ -29,10 +29,10 @@ func (h *logHandler) TextDocumentSymbol(ctx context.Context, params lsp.Document mod, err := mf.ModuleByPath(file.Dir()) if err != nil { - return symbols, fmt.Errorf("finding compatible decoder failed: %w", err) + return symbols, err } - d, err := mod.Decoder() + d, err := module.DecoderForModule(mod) if err != nil { return symbols, err } diff --git a/internal/langserver/handlers/symbols_test.go b/internal/langserver/handlers/symbols_test.go index 621b659c0..cb89023c1 100644 --- a/internal/langserver/handlers/symbols_test.go +++ b/internal/langserver/handlers/symbols_test.go @@ -5,7 +5,8 @@ import ( "testing" "github.com/hashicorp/terraform-ls/internal/langserver" - "github.com/hashicorp/terraform-ls/internal/terraform/module" + "github.com/hashicorp/terraform-ls/internal/terraform/exec" + "github.com/stretchr/testify/mock" ) func TestLangServer_symbols_basic(t *testing.T) { @@ -13,9 +14,9 @@ func TestLangServer_symbols_basic(t *testing.T) { InitPluginCache(t, tmpDir.Dir()) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: validTfMockCalls(), + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), }, }, })) diff --git a/internal/langserver/handlers/tick_reporter.go b/internal/langserver/handlers/tick_reporter.go deleted file mode 100644 index 55a8e8e26..000000000 --- a/internal/langserver/handlers/tick_reporter.go +++ /dev/null @@ -1,40 +0,0 @@ -package handlers - -import ( - "context" - "time" -) - -func newTickReporter(d time.Duration) *tickReporter { - return &tickReporter{ - t: time.NewTicker(d), - rfs: make([]reportFunc, 0), - } -} - -type reportFunc func() - -type tickReporter struct { - t *time.Ticker - rfs []reportFunc -} - -func (tr *tickReporter) AddReporter(f reportFunc) { - tr.rfs = append(tr.rfs, f) -} - -func (tr *tickReporter) StartReporting(ctx context.Context) { - go func(ctx context.Context, tr *tickReporter) { - for { - select { - case <-ctx.Done(): - tr.t.Stop() - return - case <-tr.t.C: - for _, rf := range tr.rfs { - rf() - } - } - } - }(ctx, tr) -} diff --git a/internal/terraform/datadir/datadir.go b/internal/terraform/datadir/datadir.go new file mode 100644 index 000000000..8948e3b8b --- /dev/null +++ b/internal/terraform/datadir/datadir.go @@ -0,0 +1,71 @@ +package datadir + +import ( + "path/filepath" + "strings" + + "github.com/hashicorp/terraform-ls/internal/filesystem" +) + +type DataDir struct { + ModuleManifestPath string + PluginLockFilePath string +} + +type WatchablePaths struct { + Dirs []string + ModuleManifests []string + PluginLockFiles []string +} + +func WatchableModulePaths(modPath string) *WatchablePaths { + wp := &WatchablePaths{ + Dirs: watchableModuleDirs(modPath), + ModuleManifests: make([]string, 0), + PluginLockFiles: make([]string, 0), + } + + manifestPath := filepath.Join(append([]string{modPath}, manifestPathElements...)...) + wp.ModuleManifests = append(wp.ModuleManifests, manifestPath) + + for _, pathElems := range pluginLockFilePathElements { + filePath := filepath.Join(append([]string{modPath}, pathElems...)...) + wp.PluginLockFiles = append(wp.PluginLockFiles, filePath) + } + + return wp +} + +// ModulePath strips known lock file paths to get the path +// to the (closest) module these files belong to +func ModulePath(filePath string) (string, bool) { + manifestSuffix := filepath.Join(manifestPathElements...) + if strings.HasSuffix(filePath, manifestSuffix) { + return strings.TrimSuffix(filePath, manifestSuffix), true + } + + for _, pathElems := range pluginLockFilePathElements { + suffix := filepath.Join(pathElems...) + if strings.HasSuffix(filePath, suffix) { + return strings.TrimSuffix(filePath, suffix), true + } + } + + return "", false +} + +func WalkDataDirOfModule(fs filesystem.Filesystem, modPath string) *DataDir { + dir := &DataDir{} + + path, ok := ModuleManifestFilePath(fs, modPath) + if ok { + dir.ModuleManifestPath = path + } + + path, ok = PluginLockFilePath(fs, modPath) + if ok { + dir.PluginLockFilePath = path + } + + return dir +} diff --git a/internal/terraform/module/module_manifest.go b/internal/terraform/datadir/module_manifest.go similarity index 76% rename from internal/terraform/module/module_manifest.go rename to internal/terraform/datadir/module_manifest.go index 079e80a4a..bc3bb707c 100644 --- a/internal/terraform/module/module_manifest.go +++ b/internal/terraform/datadir/module_manifest.go @@ -1,4 +1,4 @@ -package module +package datadir import ( "encoding/json" @@ -8,14 +8,19 @@ import ( "strings" version "github.com/hashicorp/go-version" + "github.com/hashicorp/terraform-ls/internal/filesystem" ) -func moduleManifestFilePath(dir string) string { - return filepath.Join( - dir, - ".terraform", - "modules", - "modules.json") +func ModuleManifestFilePath(fs filesystem.Filesystem, modulePath string) (string, bool) { + manifestPath := filepath.Join( + append([]string{modulePath}, + manifestPathElements...)...) + + fi, err := fs.Stat(manifestPath) + if err == nil && fi.Mode().IsRegular() { + return manifestPath, true + } + return "", false } // The following structs were copied from terraform's @@ -90,15 +95,16 @@ func (r *ModuleRecord) IsExternal() bool { return false } -// moduleManifest is an internal struct used only to assist in our JSON -// serialization of manifest snapshots. It should not be used for any other -// purpose. -type moduleManifest struct { +type ModuleManifest struct { rootDir string Records []ModuleRecord `json:"Modules"` } -func ParseModuleManifestFromFile(path string) (*moduleManifest, error) { +func (mm *ModuleManifest) RootDir() string { + return mm.rootDir +} + +func ParseModuleManifestFromFile(path string) (*ModuleManifest, error) { b, err := ioutil.ReadFile(path) if err != nil { return nil, err @@ -108,13 +114,17 @@ func ParseModuleManifestFromFile(path string) (*moduleManifest, error) { if err != nil { return nil, err } - mm.rootDir = trimLockFilePath(path) + rootDir, ok := ModulePath(path) + if !ok { + return nil, fmt.Errorf("failed to detect module path: %s", path) + } + mm.rootDir = filepath.Clean(rootDir) return mm, nil } -func parseModuleManifest(b []byte) (*moduleManifest, error) { - mm := moduleManifest{} +func parseModuleManifest(b []byte) (*ModuleManifest, error) { + mm := ModuleManifest{} err := json.Unmarshal(b, &mm) if err != nil { return nil, err diff --git a/internal/terraform/datadir/module_manifest_test.go b/internal/terraform/datadir/module_manifest_test.go new file mode 100644 index 000000000..fb75daedd --- /dev/null +++ b/internal/terraform/datadir/module_manifest_test.go @@ -0,0 +1,116 @@ +package datadir + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/hashicorp/go-version" +) + +func TestParseModuleManifestFromFile(t *testing.T) { + modPath := t.TempDir() + manifestDir := filepath.Join(modPath, ".terraform", "modules") + err := os.MkdirAll(manifestDir, 0755) + if err != nil { + t.Fatal(err) + } + + expectedManifest := &ModuleManifest{ + rootDir: modPath, + Records: []ModuleRecord{ + { + Key: "web_server_sg1", + SourceAddr: "terraform-aws-modules/security-group/aws//modules/http-80", + VersionStr: "3.10.0", + Version: version.Must(version.NewVersion("3.10.0")), + Dir: filepath.Join(".terraform", "modules", "web_server_sg", "terraform-aws-security-group-3.10.0", "modules", "http-80"), + }, + { + Key: "web_server_sg2", + SourceAddr: "terraform-aws-modules/security-group/aws//modules/http-80", + VersionStr: "3.10.0", + Version: version.Must(version.NewVersion("3.10.0")), + Dir: filepath.Join(".terraform", "modules", "web_server_sg", "terraform-aws-security-group-3.10.0", "modules", "http-80"), + }, + { + Dir: ".", + }, + { + Key: "local", + SourceAddr: "./nested/path", + Dir: filepath.Join("nested", "path"), + }, + }, + } + + path := filepath.Join(manifestDir, "modules.json") + err = ioutil.WriteFile(path, []byte(testManifestContent), 0755) + if err != nil { + t.Fatal(err) + } + mm, err := ParseModuleManifestFromFile(path) + if err != nil { + t.Fatal(err) + } + + opts := cmp.AllowUnexported(ModuleManifest{}) + if diff := cmp.Diff(expectedManifest, mm, opts); diff != "" { + t.Fatalf("manifest mismatch: %s", diff) + } +} + +const testManifestContent = `{ + "Modules": [ + { + "Key": "web_server_sg1", + "Source": "terraform-aws-modules/security-group/aws//modules/http-80", + "Version": "3.10.0", + "Dir": ".terraform/modules/web_server_sg/terraform-aws-security-group-3.10.0/modules/http-80" + }, + { + "Key": "web_server_sg2", + "Source": "terraform-aws-modules/security-group/aws//modules/http-80", + "Version": "3.10.0", + "Dir": ".terraform/modules/web_server_sg/terraform-aws-security-group-3.10.0/modules/something/../http-80" + }, + { + "Key": "", + "Source": "", + "Dir": "." + }, + { + "Key": "local", + "Source": "./nested/path", + "Dir": "nested/path" + } + ] +}` + +const moduleManifestRecord_external = `{ + "Key": "web_server_sg", + "Source": "terraform-aws-modules/security-group/aws//modules/http-80", + "Version": "3.10.0", + "Dir": ".terraform/modules/web_server_sg/terraform-aws-security-group-3.10.0/modules/http-80" +}` + +const moduleManifestRecord_externalDirtyPath = `{ + "Key": "web_server_sg", + "Source": "terraform-aws-modules/security-group/aws//modules/http-80", + "Version": "3.10.0", + "Dir": ".terraform/modules/web_server_sg/terraform-aws-security-group-3.10.0/modules/something/../http-80" +}` + +const moduleManifestRecord_local = `{ + "Key": "local", + "Source": "./nested/path", + "Dir": "nested/path" +}` + +const moduleManifestRecord_root = `{ + "Key": "", + "Source": "", + "Dir": "." +}` diff --git a/internal/terraform/module/module_manifest_unix_test.go b/internal/terraform/datadir/module_manifest_unix_test.go similarity index 99% rename from internal/terraform/module/module_manifest_unix_test.go rename to internal/terraform/datadir/module_manifest_unix_test.go index c024b7198..a4b4e25f1 100644 --- a/internal/terraform/module/module_manifest_unix_test.go +++ b/internal/terraform/datadir/module_manifest_unix_test.go @@ -1,6 +1,6 @@ // +build !windows -package module +package datadir import ( "encoding/json" diff --git a/internal/terraform/module/module_manifest_windows_test.go b/internal/terraform/datadir/module_manifest_windows_test.go similarity index 99% rename from internal/terraform/module/module_manifest_windows_test.go rename to internal/terraform/datadir/module_manifest_windows_test.go index 77a8071f6..b246712b6 100644 --- a/internal/terraform/module/module_manifest_windows_test.go +++ b/internal/terraform/datadir/module_manifest_windows_test.go @@ -1,4 +1,4 @@ -package module +package datadir import ( "encoding/json" diff --git a/internal/terraform/datadir/paths.go b/internal/terraform/datadir/paths.go new file mode 100644 index 000000000..152ef6b7e --- /dev/null +++ b/internal/terraform/datadir/paths.go @@ -0,0 +1,30 @@ +package datadir + +import ( + "path/filepath" + "runtime" +) + +const DataDirName = ".terraform" + +var pluginLockFilePathElements = [][]string{ + // Terraform >= 0.14 + {".terraform.lock.hcl"}, + // Terraform >= v0.13 + {DataDirName, "plugins", "selections.json"}, + // Terraform >= v0.12 + {DataDirName, "plugins", runtime.GOOS + "_" + runtime.GOARCH, "lock.json"}, +} + +var manifestPathElements = []string{ + DataDirName, "modules", "modules.json", +} + +func watchableModuleDirs(modPath string) []string { + return []string{ + filepath.Join(modPath, DataDirName), + filepath.Join(modPath, DataDirName, "modules"), + filepath.Join(modPath, DataDirName, "plugins"), + filepath.Join(modPath, DataDirName, "plugins", runtime.GOOS+"_"+runtime.GOARCH), + } +} diff --git a/internal/terraform/datadir/plugin_lock_file.go b/internal/terraform/datadir/plugin_lock_file.go new file mode 100644 index 000000000..3e3155480 --- /dev/null +++ b/internal/terraform/datadir/plugin_lock_file.go @@ -0,0 +1,19 @@ +package datadir + +import ( + "path/filepath" + + "github.com/hashicorp/terraform-ls/internal/filesystem" +) + +func PluginLockFilePath(fs filesystem.Filesystem, modPath string) (string, bool) { + for _, pathElems := range pluginLockFilePathElements { + fullPath := filepath.Join(append([]string{modPath}, pathElems...)...) + fi, err := fs.Stat(fullPath) + if err == nil && fi.Mode().IsRegular() { + return fullPath, true + } + } + + return "", false +} diff --git a/internal/terraform/exec/exec.go b/internal/terraform/exec/exec.go index e0b3a8b1f..792a3cfba 100644 --- a/internal/terraform/exec/exec.go +++ b/internal/terraform/exec/exec.go @@ -16,6 +16,8 @@ import ( var defaultExecTimeout = 30 * time.Second +type ctxKey string + type Executor struct { tf *tfexec.Terraform timeout time.Duration diff --git a/internal/terraform/exec/exec_mock.go b/internal/terraform/exec/exec_mock.go index 4999efeb1..73f4173d3 100644 --- a/internal/terraform/exec/exec_mock.go +++ b/internal/terraform/exec/exec_mock.go @@ -1,12 +1,34 @@ package exec import ( + "context" + "fmt" + exec_mock "github.com/hashicorp/terraform-ls/internal/terraform/exec/mock" "github.com/stretchr/testify/mock" ) -func NewMockExecutor(calls []*mock.Call) ExecutorFactory { - return func(string, string) (TerraformExecutor, error) { +type TerraformMockCalls struct { + PerWorkDir map[string][]*mock.Call + AnyWorkDir []*mock.Call +} + +func NewMockExecutor(calls *TerraformMockCalls) ExecutorFactory { + return func(workDir string, execPath string) (TerraformExecutor, error) { + if calls == nil { + return nil, fmt.Errorf("%s: no mock calls defined", workDir) + } + mockCalls := calls.AnyWorkDir + if len(calls.PerWorkDir) > 0 { + mc, ok := calls.PerWorkDir[workDir] + if ok { + mockCalls = mc + } + } + if len(mockCalls) == 0 { + return nil, fmt.Errorf("%s: no mock calls available for this workdir", workDir) + } + me := &exec_mock.Executor{} firstCalls := []*mock.Call{ { @@ -15,7 +37,19 @@ func NewMockExecutor(calls []*mock.Call) ExecutorFactory { Repeatability: 1, }, } - me.ExpectedCalls = append(firstCalls, calls...) + + me.ExpectedCalls = append(firstCalls, mockCalls...) return me, nil } } + +var ctxExecutorFactory = ctxKey("executor factory") + +func ExecutorFactoryFromContext(ctx context.Context) (ExecutorFactory, bool) { + f, ok := ctx.Value(ctxExecutorFactory).(ExecutorFactory) + return f, ok +} + +func WithExecutorFactory(ctx context.Context, f ExecutorFactory) context.Context { + return context.WithValue(ctx, ctxExecutorFactory, f) +} diff --git a/internal/terraform/exec/exec_opts.go b/internal/terraform/exec/exec_opts.go new file mode 100644 index 000000000..37cae5267 --- /dev/null +++ b/internal/terraform/exec/exec_opts.go @@ -0,0 +1,23 @@ +package exec + +import ( + "context" + "time" +) + +type ExecutorOpts struct { + ExecPath string + ExecLogPath string + Timeout time.Duration +} + +var ctxExecOpts = ctxKey("executor opts") + +func ExecutorOptsFromContext(ctx context.Context) (*ExecutorOpts, bool) { + opts, ok := ctx.Value(ctxExecOpts).(*ExecutorOpts) + return opts, ok +} + +func WithExecutorOpts(ctx context.Context, opts *ExecutorOpts) context.Context { + return context.WithValue(ctx, ctxExecOpts, opts) +} diff --git a/internal/terraform/module/file.go b/internal/terraform/module/file.go deleted file mode 100644 index 388469d20..000000000 --- a/internal/terraform/module/file.go +++ /dev/null @@ -1,43 +0,0 @@ -package module - -import ( - "fmt" - "os" -) - -func findFile(paths []string) (File, error) { - var lf File - var err error - - for _, path := range paths { - lf, err = newFile(path) - if err == nil { - return lf, nil - } - if !os.IsNotExist(err) { - return nil, err - } - } - - return nil, err -} - -type file struct { - path string -} - -func (f *file) Path() string { - return f.path -} - -func newFile(path string) (File, error) { - fi, err := os.Stat(path) - if err != nil { - return nil, err - } - if fi.IsDir() { - return nil, fmt.Errorf("expected %s to be a file, not a dir", path) - } - - return &file{path: path}, nil -} diff --git a/internal/terraform/module/module.go b/internal/terraform/module/module.go index 0156f42e9..d0c771617 100644 --- a/internal/terraform/module/module.go +++ b/internal/terraform/module/module.go @@ -1,674 +1,233 @@ package module import ( - "context" - "errors" - "fmt" - "io/ioutil" "log" - "os" "path/filepath" - "strings" "sync" - "time" - "github.com/hashicorp/go-multierror" "github.com/hashicorp/go-version" - "github.com/hashicorp/hcl-lang/decoder" - "github.com/hashicorp/hcl-lang/schema" "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/hclsyntax" tfjson "github.com/hashicorp/terraform-json" "github.com/hashicorp/terraform-ls/internal/filesystem" - "github.com/hashicorp/terraform-ls/internal/schemas" - "github.com/hashicorp/terraform-ls/internal/terraform/discovery" - "github.com/hashicorp/terraform-ls/internal/terraform/exec" - tfschema "github.com/hashicorp/terraform-schema/schema" + "github.com/hashicorp/terraform-ls/internal/terraform/datadir" ) type module struct { path string + fs filesystem.Filesystem logger *log.Logger - // loading - isLoading bool - isLoadingMu *sync.RWMutex - loadingDone <-chan struct{} - cancelLoading context.CancelFunc - loadErr error - loadErrMu *sync.RWMutex - - // module cache - moduleMu *sync.RWMutex - moduleManifestFile File - moduleManifest *moduleManifest - - // plugin (provider schema) cache - pluginMu *sync.RWMutex - pluginLockFile File - providerSchema *tfjson.ProviderSchemas - providerSchemaMu *sync.RWMutex - providerVersions map[string]*version.Version - - // terraform executor - tfLoadingDone bool - tfLoadingMu *sync.RWMutex - tfExec exec.TerraformExecutor - tfNewExecutor exec.ExecutorFactory - tfExecPath string - tfExecTimeout time.Duration - tfExecLogPath string - - // terraform discovery - tfDiscoFunc discovery.DiscoveryFunc - tfDiscoErr error - tfVersion *version.Version - tfVersionErr error - - // core schema - coreSchema *schema.BodySchema - coreSchemaMu *sync.RWMutex - - // decoder - isParsed bool - isParsedMu *sync.RWMutex - pFilesMap map[string]*hcl.File - parsedDiags map[string]hcl.Diagnostics - parserMu *sync.RWMutex - filesystem filesystem.Filesystem + // module manifest + modManifest *datadir.ModuleManifest + modManifestErr error + modManifestMu *sync.RWMutex + modManigestState OpState + modManifestStateMu *sync.RWMutex + + // provider schema + providerSchema *tfjson.ProviderSchemas + providerSchemaErr error + providerSchemaMu *sync.RWMutex + providerSchemaState OpState + providerSchemaStateMu *sync.RWMutex + + // terraform exec path + tfExecPath string + tfExecPathMu *sync.RWMutex + + // terraform version + tfVersion *version.Version + tfVersionErr error + tfVersionMu *sync.RWMutex + tfVersionState OpState + tfVersionStateMu *sync.RWMutex + + // provider versions + providerVersions map[string]*version.Version + providerVersionsMu *sync.RWMutex + + // config (HCL) parser + parsedFiles map[string]*hcl.File + parsingErr error + parserMu *sync.RWMutex + parserState OpState + parserStateMu *sync.RWMutex + + // module diagnostics + diags map[string]hcl.Diagnostics + diagsMu *sync.RWMutex } func newModule(fs filesystem.Filesystem, dir string) *module { return &module{ - path: dir, - filesystem: fs, - logger: defaultLogger, - isLoadingMu: &sync.RWMutex{}, - loadErrMu: &sync.RWMutex{}, - moduleMu: &sync.RWMutex{}, - pluginMu: &sync.RWMutex{}, - providerSchemaMu: &sync.RWMutex{}, - tfLoadingMu: &sync.RWMutex{}, - coreSchema: tfschema.UniversalCoreModuleSchema(), - coreSchemaMu: &sync.RWMutex{}, - isParsedMu: &sync.RWMutex{}, - pFilesMap: make(map[string]*hcl.File, 0), - providerVersions: make(map[string]*version.Version, 0), - parserMu: &sync.RWMutex{}, - } -} - -var defaultLogger = log.New(ioutil.Discard, "", 0) - -func NewModule(ctx context.Context, fs filesystem.Filesystem, dir string) (Module, error) { - m := newModule(fs, dir) - - d := &discovery.Discovery{} - m.tfDiscoFunc = d.LookPath - - m.tfNewExecutor = exec.NewExecutor - - err := m.discoverCaches(ctx, dir) - if err != nil { - return m, err - } - - return m, m.load(ctx) -} - -func (m *module) discoverCaches(ctx context.Context, dir string) error { - var errs *multierror.Error - err := m.discoverPluginCache(dir) - if err != nil { - errs = multierror.Append(errs, err) - } - - err = m.discoverModuleCache(dir) - if err != nil { - errs = multierror.Append(errs, err) - } - - return errs.ErrorOrNil() -} - -func (m *module) WasInitialized() (bool, error) { - tfDirPath := filepath.Join(m.Path(), ".terraform") - - f, err := m.filesystem.Open(tfDirPath) - if err != nil { - return false, err - } - defer f.Close() - fi, err := f.Stat() - if err != nil { - return false, err - } - if !fi.IsDir() { - return false, fmt.Errorf("%s is not a directory", tfDirPath) - } - - return true, nil -} - -func (m *module) discoverPluginCache(dir string) error { - m.pluginMu.Lock() - defer m.pluginMu.Unlock() - - lockPaths := pluginLockFilePaths(dir) - lf, err := findFile(lockPaths) - if err != nil { - if os.IsNotExist(err) { - m.logger.Printf("no plugin cache found: %s", err.Error()) - return nil - } + path: dir, + fs: fs, + logger: defaultLogger, - return fmt.Errorf("unable to calculate hash: %w", err) + modManifestMu: &sync.RWMutex{}, + modManifestStateMu: &sync.RWMutex{}, + providerSchemaMu: &sync.RWMutex{}, + providerSchemaStateMu: &sync.RWMutex{}, + providerVersions: make(map[string]*version.Version, 0), + providerVersionsMu: &sync.RWMutex{}, + tfVersionMu: &sync.RWMutex{}, + tfVersionStateMu: &sync.RWMutex{}, + tfExecPathMu: &sync.RWMutex{}, + parsedFiles: make(map[string]*hcl.File, 0), + parserMu: &sync.RWMutex{}, + parserStateMu: &sync.RWMutex{}, + diagsMu: &sync.RWMutex{}, } - m.pluginLockFile = lf - return nil } -func (m *module) discoverModuleCache(dir string) error { - m.moduleMu.Lock() - defer m.moduleMu.Unlock() - - lf, err := newFile(moduleManifestFilePath(dir)) - if err != nil { - if os.IsNotExist(err) { - m.logger.Printf("no module manifest file found: %s", err.Error()) - return nil - } - - return fmt.Errorf("unable to calculate hash: %w", err) - } - m.moduleManifestFile = lf - return nil -} - -func (m *module) Modules() []ModuleRecord { - m.moduleMu.Lock() - defer m.moduleMu.Unlock() - if m.moduleManifest == nil { - return []ModuleRecord{} - } - - return m.moduleManifest.Records -} - -func (m *module) SetLogger(logger *log.Logger) { - m.logger = logger -} - -func (m *module) StartLoading() error { - if !m.IsLoadingDone() { - return fmt.Errorf("module is already being loaded") - } - ctx, cancelFunc := context.WithCancel(context.Background()) - m.cancelLoading = cancelFunc - m.loadingDone = ctx.Done() - - go func(ctx context.Context) { - m.setLoadErr(m.load(ctx)) - }(ctx) - return nil +func NewModule(fs filesystem.Filesystem, dir string) Module { + return newModule(fs, dir) } -func (m *module) CancelLoading() { - if !m.IsLoadingDone() && m.cancelLoading != nil { - m.cancelLoading() - } - m.setLoadingState(false) -} - -func (m *module) LoadingDone() <-chan struct{} { - return m.loadingDone -} - -func (m *module) load(ctx context.Context) error { - var errs *multierror.Error - defer m.CancelLoading() - - // reset internal loading state - m.setLoadingState(true) - - // The following operations have to happen in a particular order - // as they depend on the internal state as mutated by each operation - - err := m.UpdateModuleManifest(m.moduleManifestFile) - errs = multierror.Append(errs, err) - - err = m.discoverTerraformExecutor(ctx) - m.tfDiscoErr = err - errs = multierror.Append(errs, err) - - err = m.discoverTerraformVersion(ctx) - m.tfVersionErr = err - errs = multierror.Append(errs, err) - - err = m.findAndSetCoreSchema() +func (m *module) HasOpenFiles() bool { + openFiles, err := m.fs.HasOpenFiles(m.Path()) if err != nil { - m.logger.Printf("%s: %s - falling back to universal schema", + m.logger.Printf("%s: failed to check whether module has open files: %s", m.Path(), err) } - - err = m.UpdateProviderSchemaCache(ctx, m.pluginLockFile) - errs = multierror.Append(errs, err) - - m.logger.Printf("loading of module %s finished: %s", - m.Path(), errs) - return errs.ErrorOrNil() -} - -func (m *module) setLoadingState(isLoading bool) { - m.isLoadingMu.Lock() - defer m.isLoadingMu.Unlock() - m.isLoading = isLoading + return openFiles } -func (m *module) IsLoadingDone() bool { - m.isLoadingMu.RLock() - defer m.isLoadingMu.RUnlock() - return !m.isLoading -} - -func (m *module) discoverTerraformExecutor(ctx context.Context) error { - defer func() { - m.setTfDiscoveryFinished(true) - }() - - tfPath := m.tfExecPath - if tfPath == "" { - var err error - tfPath, err = m.tfDiscoFunc() - if err != nil { - return err - } - } - - tf, err := m.tfNewExecutor(m.path, tfPath) - if err != nil { - return err - } - - tf.SetLogger(m.logger) - - if m.tfExecLogPath != "" { - tf.SetExecLogPath(m.tfExecLogPath) - } - - if m.tfExecTimeout != 0 { - tf.SetTimeout(m.tfExecTimeout) - } - - m.tfExec = tf - - return nil -} - -func (m *module) ExecuteTerraformInit(ctx context.Context) error { - if !m.IsTerraformAvailable() { - if err := m.discoverTerraformExecutor(ctx); err != nil { - return err - } - } - - return m.tfExec.Init(ctx) -} - -func (m *module) ExecuteTerraformValidate(ctx context.Context) (map[string]hcl.Diagnostics, error) { - diagsMap := make(map[string]hcl.Diagnostics) - - if !m.IsTerraformAvailable() { - if err := m.discoverTerraformExecutor(ctx); err != nil { - return diagsMap, err - } - } - - if !m.IsParsed() { - if err := m.ParseFiles(); err != nil { - return diagsMap, err - } - } - - // an entry for each file should exist, even if there are no diags - for filename := range m.parsedFiles() { - diagsMap[filename] = make(hcl.Diagnostics, 0) - } - // since validation applies to linked modules, create an entry for all - // files of linked modules - for _, mod := range m.moduleManifest.Records { - if mod.IsRoot() { - // skip root module - continue - } - if mod.IsExternal() { - // skip external module - continue - } - - absPath := filepath.Join(m.moduleManifest.rootDir, mod.Dir) - infos, err := m.filesystem.ReadDir(absPath) - if err != nil { - return diagsMap, fmt.Errorf("failed to read module at %q: %w", absPath, err) - } - - for _, info := range infos { - if info.IsDir() { - // We only care about files - continue - } - - name := info.Name() - if !strings.HasSuffix(name, ".tf") || IsIgnoredFile(name) { - continue - } - - // map entries are relative to the parent module path - filename := filepath.Join(mod.Dir, name) - - diagsMap[filename] = make(hcl.Diagnostics, 0) - } - } - - validationDiags, err := m.tfExec.Validate(ctx) - if err != nil { - return diagsMap, err - } - - // tfjson.Diagnostic is a conversion of an internal diag to terraform core, - // tfdiags, which is effectively based on hcl.Diagnostic. - // This process is really just converting it back to hcl.Diagnotic - // since it is the defacto diagnostic type for our codebase currently - // https://github.com/hashicorp/terraform/blob/ae025248cc0712bf53c675dc2fe77af4276dd5cc/command/validate.go#L138 - for _, d := range validationDiags { - // the diagnostic must be tied to a file to exist in the map - if d.Range == nil || d.Range.Filename == "" { - continue - } - - diags := diagsMap[d.Range.Filename] - - var severity hcl.DiagnosticSeverity - if d.Severity == "error" { - severity = hcl.DiagError - } else if d.Severity == "warning" { - severity = hcl.DiagWarning - } - - diags = append(diags, &hcl.Diagnostic{ - Severity: severity, - Summary: d.Summary, - Detail: d.Detail, - Subject: &hcl.Range{ - Filename: d.Range.Filename, - Start: hcl.Pos(d.Range.Start), - End: hcl.Pos(d.Range.End), - }, - }) - diagsMap[d.Range.Filename] = diags - } - - return diagsMap, nil +func (m *module) SetTerraformVersion(v *version.Version, err error) { + m.tfVersionMu.Lock() + defer m.tfVersionMu.Unlock() + m.tfVersion = v + m.tfVersionErr = err } -func (m *module) discoverTerraformVersion(ctx context.Context) error { - if m.tfExec == nil { - return errors.New("no terraform executor - unable to read version") - } - - version, providerVersions, err := m.tfExec.Version(ctx) - if err != nil { - return err - } - m.logger.Printf("Terraform version %s found at %s for %s", version, - m.tfExec.GetExecPath(), m.Path()) - m.tfVersion = version - - m.providerVersions = providerVersions - - return nil +func (m *module) TerraformVersion() (*version.Version, error) { + m.tfVersionMu.RLock() + defer m.tfVersionMu.RUnlock() + return m.tfVersion, m.tfVersionErr } -func (m *module) findAndSetCoreSchema() error { - if m.tfVersion == nil { - return errors.New("unable to find core schema without version") - } - - coreSchema, err := tfschema.CoreModuleSchemaForVersion(m.tfVersion) - if err != nil { - return err - } - - m.coreSchemaMu.Lock() - m.coreSchema = coreSchema - m.coreSchemaMu.Unlock() - - return nil +func (m *module) SetProviderVersions(pv map[string]*version.Version) { + m.providerVersionsMu.Lock() + defer m.providerVersionsMu.Unlock() + m.providerVersions = pv } -func (m *module) LoadError() error { - m.loadErrMu.RLock() - defer m.loadErrMu.RUnlock() - return m.loadErr +func (m *module) ProviderVersions() map[string]*version.Version { + m.providerVersionsMu.RLock() + defer m.providerVersionsMu.RUnlock() + return m.providerVersions } -func (m *module) setLoadErr(err error) { - m.loadErrMu.Lock() - defer m.loadErrMu.Unlock() - m.loadErr = err +func (m *module) TerraformVersionState() OpState { + m.tfVersionMu.RLock() + defer m.tfVersionMu.RUnlock() + return m.tfVersionState } -func (m *module) Path() string { - return m.path +func (m *module) SetTerraformVersionState(state OpState) { + m.tfVersionMu.Lock() + defer m.tfVersionMu.Unlock() + m.tfVersionState = state } -func (m *module) MatchesPath(path string) bool { - return filepath.Clean(m.path) == filepath.Clean(path) +func (m *module) SetModuleManifest(manifest *datadir.ModuleManifest, err error) { + m.modManifestMu.Lock() + defer m.modManifestMu.Unlock() + m.modManifest = manifest + m.modManifestErr = err } -// HumanReadablePath helps display shorter, but still relevant paths -func (m *module) HumanReadablePath(rootDir string) string { - if rootDir == "" { - return m.path - } - - // absolute paths can be too long for UI/messages, - // so we just display relative to root dir - relDir, err := filepath.Rel(rootDir, m.path) - if err != nil { - return m.path - } - - if relDir == "." { - // Name of the root dir is more helpful than "." - return filepath.Base(rootDir) - } - - return relDir +func (m *module) ModuleManifestState() OpState { + m.modManifestMu.RLock() + defer m.modManifestMu.RUnlock() + return m.modManigestState } -func (m *module) UpdateModuleManifest(lockFile File) error { - m.moduleMu.Lock() - defer m.moduleMu.Unlock() - - if lockFile == nil { - m.logger.Printf("ignoring module update as no lock file was found for %s", m.Path()) - return nil - } - - m.moduleManifestFile = lockFile - - mm, err := ParseModuleManifestFromFile(lockFile.Path()) - if err != nil { - return fmt.Errorf("failed to update module manifest: %w", err) - } - - m.moduleManifest = mm - m.logger.Printf("updated module manifest - %d references parsed for %s", - len(mm.Records), m.Path()) - return nil +func (m *module) SetModuleManifestParsingState(state OpState) { + m.modManifestMu.Lock() + defer m.modManifestMu.Unlock() + m.modManigestState = state } -func (m *module) DecoderWithSchema(schema *schema.BodySchema) (*decoder.Decoder, error) { - d, err := m.Decoder() - if err != nil { - return nil, err - } - - d.SetSchema(schema) - - return d, nil +func (m *module) SetProviderSchemas(ps *tfjson.ProviderSchemas, err error) { + m.providerSchemaMu.Lock() + defer m.providerSchemaMu.Unlock() + m.providerSchema = ps + m.providerSchemaErr = err } -func (m *module) Decoder() (*decoder.Decoder, error) { - d := decoder.NewDecoder() - - for name, f := range m.parsedFiles() { - err := d.LoadFile(name, f) - if err != nil { - return nil, fmt.Errorf("failed to load a file: %w", err) - } - } - return d, nil +func (m *module) ProviderSchema() (*tfjson.ProviderSchemas, error) { + m.providerSchemaMu.RLock() + defer m.providerSchemaMu.RUnlock() + return m.providerSchema, m.providerSchemaErr } -func (m *module) IsProviderSchemaLoaded() bool { +func (m *module) ProviderSchemaState() OpState { m.providerSchemaMu.RLock() defer m.providerSchemaMu.RUnlock() - return m.providerSchema != nil + return m.providerSchemaState } -func (m *module) IsParsed() bool { - m.isParsedMu.RLock() - defer m.isParsedMu.RUnlock() - return m.isParsed +func (m *module) SetProviderSchemaObtainingState(state OpState) { + m.providerSchemaMu.Lock() + defer m.providerSchemaMu.Unlock() + m.providerSchemaState = state } -func (m *module) setIsParsed(parsed bool) { - m.isParsedMu.Lock() - defer m.isParsedMu.Unlock() - m.isParsed = parsed +func (m *module) ParsedFiles() (map[string]*hcl.File, error) { + m.parserMu.RLock() + defer m.parserMu.RUnlock() + return m.parsedFiles, m.parsingErr } -func (m *module) ParseFiles() error { +func (m *module) SetParsedFiles(files map[string]*hcl.File, err error) { m.parserMu.Lock() defer m.parserMu.Unlock() - - files := make(map[string]*hcl.File, 0) - diags := make(map[string]hcl.Diagnostics, 0) - - infos, err := m.filesystem.ReadDir(m.Path()) - if err != nil { - return fmt.Errorf("failed to read module at %q: %w", m.Path(), err) - } - - for _, info := range infos { - if info.IsDir() { - // We only care about files - continue - } - - name := info.Name() - if !strings.HasSuffix(name, ".tf") || IsIgnoredFile(name) { - continue - } - - // TODO: overrides - - fullPath := filepath.Join(m.Path(), name) - - src, err := m.filesystem.ReadFile(fullPath) - if err != nil { - return fmt.Errorf("failed to read %q: %s", name, err) - } - - m.logger.Printf("parsing file %q", name) - f, pDiags := hclsyntax.ParseConfig(src, name, hcl.InitialPos) - diags[name] = pDiags - if f != nil { - files[name] = f - } - } - - m.pFilesMap = files - m.parsedDiags = diags - m.setIsParsed(true) - - return nil + m.parsedFiles = files + m.parsingErr = err } -func (m *module) ParsedDiagnostics() map[string]hcl.Diagnostics { - m.parserMu.Lock() - defer m.parserMu.Unlock() - return m.parsedDiags +func (m *module) SetDiagnostics(diags map[string]hcl.Diagnostics) { + m.diagsMu.Lock() + defer m.diagsMu.Unlock() + m.diags = diags } -func (m *module) parsedFiles() map[string]*hcl.File { +func (m *module) ConfigParsingState() OpState { m.parserMu.RLock() defer m.parserMu.RUnlock() - - return m.pFilesMap + return m.parserState } -func (m *module) MergedSchema() (*schema.BodySchema, error) { - m.coreSchemaMu.RLock() - defer m.coreSchemaMu.RUnlock() - - if !m.IsParsed() { - err := m.ParseFiles() - if err != nil { - return nil, err - } - } - - ps, vOut, err := schemas.PreloadedProviderSchemas() - if err != nil { - return nil, err - } - providerVersions := vOut.Providers - tfVersion := vOut.Core - - if m.IsProviderSchemaLoaded() { - m.providerSchemaMu.RLock() - defer m.providerSchemaMu.RUnlock() - ps = m.providerSchema - providerVersions = m.providerVersions - tfVersion = m.tfVersion - } - - if ps == nil { - m.logger.Print("provider schemas is nil... skipping merge with core schema") - return m.coreSchema, nil - } - - sm := tfschema.NewSchemaMerger(m.coreSchema) - sm.SetCoreVersion(tfVersion) - sm.SetParsedFiles(m.parsedFiles()) - - err = sm.SetProviderVersions(providerVersions) - if err != nil { - return nil, err - } +func (m *module) SetConfigParsingState(state OpState) { + m.parserMu.Lock() + defer m.parserMu.Unlock() + m.parserState = state +} - return sm.MergeWithJsonProviderSchemas(ps) +func (m *module) ModuleManifest() (*datadir.ModuleManifest, error) { + m.modManifestMu.RLock() + defer m.modManifestMu.RUnlock() + return m.modManifest, m.modManifestErr } -// IsIgnoredFile returns true if the given filename (which must not have a -// directory path ahead of it) should be ignored as e.g. an editor swap file. -func IsIgnoredFile(name string) bool { - return strings.HasPrefix(name, ".") || // Unix-like hidden files - strings.HasSuffix(name, "~") || // vim - strings.HasPrefix(name, "#") && strings.HasSuffix(name, "#") // emacs +func (m *module) ModuleCalls() []datadir.ModuleRecord { + m.modManifestMu.RLock() + defer m.modManifestMu.RUnlock() + if m.modManifest == nil { + return []datadir.ModuleRecord{} + } + return m.modManifest.Records } -func (m *module) ReferencesModulePath(path string) bool { - m.moduleMu.Lock() - defer m.moduleMu.Unlock() - if m.moduleManifest == nil { +func (m *module) CallsModule(path string) bool { + m.modManifestMu.RLock() + defer m.modManifestMu.RUnlock() + if m.modManifest == nil { return false } - for _, mod := range m.moduleManifest.Records { + for _, mod := range m.modManifest.Records { if mod.IsRoot() { // skip root module, as that's tracked separately continue @@ -677,7 +236,7 @@ func (m *module) ReferencesModulePath(path string) bool { // skip external modules as these shouldn't be modified from cache continue } - absPath := filepath.Join(m.moduleManifest.rootDir, mod.Dir) + absPath := filepath.Join(m.modManifest.RootDir(), mod.Dir) if pathEquals(absPath, path) { return true } @@ -686,97 +245,47 @@ func (m *module) ReferencesModulePath(path string) bool { return false } -func (m *module) TerraformFormatter() (exec.Formatter, error) { - if !m.HasTerraformDiscoveryFinished() { - return nil, fmt.Errorf("terraform is not loaded yet") - } - - if !m.IsTerraformAvailable() { - return nil, fmt.Errorf("terraform is not available") - } - - return m.tfExec.Format, nil -} - -func (m *module) HasTerraformDiscoveryFinished() bool { - m.tfLoadingMu.RLock() - defer m.tfLoadingMu.RUnlock() - return m.tfLoadingDone +func (m *module) SetLogger(logger *log.Logger) { + m.logger = logger } -func (m *module) setTfDiscoveryFinished(isLoaded bool) { - m.tfLoadingMu.Lock() - defer m.tfLoadingMu.Unlock() - m.tfLoadingDone = isLoaded +func (m *module) Path() string { + return m.path } -func (m *module) IsTerraformAvailable() bool { - return m.HasTerraformDiscoveryFinished() && m.tfExec != nil +func (m *module) MatchesPath(path string) bool { + return pathEquals(m.path, path) } -func (m *module) UpdateProviderSchemaCache(ctx context.Context, lockFile File) error { - m.pluginMu.Lock() - defer m.pluginMu.Unlock() - - if !m.IsTerraformAvailable() { - return fmt.Errorf("cannot update provider schema as terraform is unavailable") - } - - if lockFile == nil { - m.logger.Printf("ignoring provider schema update as no lock file was provided for %s", - m.Path()) - return nil +// HumanReadablePath helps display shorter, but still relevant paths +func (m *module) HumanReadablePath(rootDir string) string { + if rootDir == "" { + return m.path } - m.pluginLockFile = lockFile - - schemas, err := m.tfExec.ProviderSchemas(ctx) + // absolute paths can be too long for UI/messages, + // so we just display relative to root dir + relDir, err := filepath.Rel(rootDir, m.path) if err != nil { - return err + return m.path } - m.providerSchemaMu.Lock() - m.providerSchema = schemas - m.providerSchemaMu.Unlock() - - return nil -} - -func (m *module) PathsToWatch() []string { - m.pluginMu.RLock() - m.moduleMu.RLock() - defer m.moduleMu.RUnlock() - defer m.pluginMu.RUnlock() - - files := make([]string, 0) - if m.pluginLockFile != nil { - files = append(files, m.pluginLockFile.Path()) - } - if m.moduleManifestFile != nil { - files = append(files, m.moduleManifestFile.Path()) + if relDir == "." { + // Name of the root dir is more helpful than "." + return filepath.Base(rootDir) } - return files + return relDir } -func (m *module) IsKnownModuleManifestFile(path string) bool { - m.moduleMu.RLock() - defer m.moduleMu.RUnlock() - - if m.moduleManifestFile == nil { - return false - } - - return pathEquals(m.moduleManifestFile.Path(), path) +func (m *module) TerraformExecPath() string { + m.tfExecPathMu.RLock() + defer m.tfExecPathMu.RUnlock() + return m.tfExecPath } -func (m *module) IsKnownPluginLockFile(path string) bool { - m.pluginMu.RLock() - defer m.pluginMu.RUnlock() - - if m.pluginLockFile == nil { - return false - } - - return pathEquals(m.pluginLockFile.Path(), path) +func (m *module) Diagnostics() map[string]hcl.Diagnostics { + m.diagsMu.RLock() + defer m.diagsMu.RUnlock() + return m.diags } diff --git a/internal/terraform/module/module_loader.go b/internal/terraform/module/module_loader.go new file mode 100644 index 000000000..46fada294 --- /dev/null +++ b/internal/terraform/module/module_loader.go @@ -0,0 +1,163 @@ +package module + +import ( + "context" + "log" + "runtime" + "sync/atomic" + + "github.com/hashicorp/terraform-ls/internal/terraform/exec" +) + +type moduleLoader struct { + queue moduleOpsQueue + nonPrioParallelism int64 + prioParallelism int64 + logger *log.Logger + tfExecOpts *exec.ExecutorOpts + opsToDispatch chan ModuleOperation + + loadingCount *int64 + prioLoadingCount *int64 +} + +func newModuleLoader() *moduleLoader { + nonPrioParallelism := 2 * runtime.NumCPU() + prioParallelism := 1 * runtime.NumCPU() + + plc, lc := int64(0), int64(0) + ml := &moduleLoader{ + queue: newModuleOpsQueue(), + logger: defaultLogger, + nonPrioParallelism: int64(nonPrioParallelism), + prioParallelism: int64(prioParallelism), + opsToDispatch: make(chan ModuleOperation, 1), + loadingCount: &lc, + prioLoadingCount: &plc, + } + + return ml +} + +func (ml *moduleLoader) SetLogger(logger *log.Logger) { + ml.logger = logger +} + +func (ml *moduleLoader) Start(ctx context.Context) { + go ml.run(ctx) +} + +func (ml *moduleLoader) run(ctx context.Context) { + for { + select { + case <-ctx.Done(): + ml.logger.Println("Cancelling module loader...") + return + case nextOp, ok := <-ml.opsToDispatch: + if !ok { + ml.logger.Println("Failed to get next operation") + return + } + + if nextOp.Module.HasOpenFiles() && ml.prioCapacity() > 0 { + atomic.AddInt64(ml.prioLoadingCount, 1) + mod := ml.queue.PopOp() + go func(ml *moduleLoader) { + defer atomic.AddInt64(ml.prioLoadingCount, -1) + ml.executeModuleOp(ctx, mod) + }(ml) + } else if ml.nonPrioCapacity() > 0 { + atomic.AddInt64(ml.loadingCount, 1) + mod := ml.queue.PopOp() + go func(ml *moduleLoader) { + defer atomic.AddInt64(ml.loadingCount, -1) + ml.executeModuleOp(ctx, mod) + }(ml) + } + } + } +} + +func (ml *moduleLoader) tryDispatchingModuleOp() { + totalCapacity := ml.nonPrioCapacity() + ml.prioCapacity() + opsInQueue := ml.queue.Len() + + // Keep scheduling work from queue if we have capacity + if opsInQueue > 0 && totalCapacity > 0 { + item := ml.queue.Peek() + nextModOp := item.(ModuleOperation) + ml.opsToDispatch <- nextModOp + } +} + +func (ml *moduleLoader) prioCapacity() int64 { + return ml.prioParallelism - atomic.LoadInt64(ml.prioLoadingCount) +} + +func (ml *moduleLoader) nonPrioCapacity() int64 { + return ml.prioParallelism - atomic.LoadInt64(ml.loadingCount) +} + +func (ml *moduleLoader) executeModuleOp(ctx context.Context, modOp ModuleOperation) { + ml.logger.Printf("executing %q for %s", modOp.Type, modOp.Module.Path()) + // TODO: Report progress in % for each op based on queue length + defer ml.logger.Printf("finished %q for %s", modOp.Type, modOp.Module.Path()) + defer modOp.markAsDone() + defer ml.tryDispatchingModuleOp() + + switch modOp.Type { + case OpTypeGetTerraformVersion: + GetTerraformVersion(ctx, modOp.Module) + return + case OpTypeObtainSchema: + ObtainSchema(ctx, modOp.Module) + return + case OpTypeParseConfiguration: + ParseConfiguration(modOp.Module) + return + case OpTypeParseModuleManifest: + ParseModuleManifest(modOp.Module) + return + } + + ml.logger.Printf("%s: unknown operation (%#v) for module operation", + modOp.Module.Path(), modOp.Type) +} + +func (ml *moduleLoader) EnqueueModuleOp(modOp ModuleOperation) { + m := modOp.Module + mod := m.(*module) + + ml.logger.Printf("ML: enqueing %q module operation: %s", modOp.Type, mod.Path()) + + switch modOp.Type { + case OpTypeGetTerraformVersion: + if mod.TerraformVersionState() == OpStateQueued { + // avoid enqueuing duplicate operation + return + } + mod.SetTerraformVersionState(OpStateQueued) + case OpTypeObtainSchema: + if mod.ProviderSchemaState() == OpStateQueued { + // avoid enqueuing duplicate operation + return + } + mod.SetProviderSchemaObtainingState(OpStateQueued) + case OpTypeParseConfiguration: + if mod.ConfigParsingState() == OpStateQueued { + // avoid enqueuing duplicate operation + return + } + mod.SetConfigParsingState(OpStateQueued) + case OpTypeParseModuleManifest: + if mod.ModuleManifestState() == OpStateQueued { + // avoid enqueuing duplicate operation + return + } + mod.SetModuleManifestParsingState(OpStateQueued) + } + + ml.queue.PushOp(modOp) + + ml.tryDispatchingModuleOp() +} diff --git a/internal/terraform/module/module_manager.go b/internal/terraform/module/module_manager.go index d37f81bdc..c0920bba6 100644 --- a/internal/terraform/module/module_manager.go +++ b/internal/terraform/module/module_manager.go @@ -4,354 +4,238 @@ import ( "context" "fmt" "log" - "os" "path/filepath" - "runtime" - "strings" - "time" - "github.com/gammazero/workerpool" + "github.com/hashicorp/go-version" "github.com/hashicorp/hcl-lang/schema" + tfjson "github.com/hashicorp/terraform-json" "github.com/hashicorp/terraform-ls/internal/filesystem" - "github.com/hashicorp/terraform-ls/internal/terraform/discovery" - "github.com/hashicorp/terraform-ls/internal/terraform/exec" + "github.com/hashicorp/terraform-ls/internal/schemas" + tfschema "github.com/hashicorp/terraform-schema/schema" ) type moduleManager struct { - modules []*module - newModule ModuleFactory - filesystem filesystem.Filesystem + modules []*module + fs filesystem.Filesystem + loader *moduleLoader syncLoading bool - workerPool *workerpool.WorkerPool + cancelFunc context.CancelFunc logger *log.Logger - - // terraform discovery - tfDiscoFunc discovery.DiscoveryFunc - - // terraform executor - tfNewExecutor exec.ExecutorFactory - tfExecPath string - tfExecTimeout time.Duration - tfExecLogPath string } -func NewModuleManager(fs filesystem.Filesystem) ModuleManager { - return newModuleManager(fs) -} - -func newModuleManager(fs filesystem.Filesystem) *moduleManager { - d := &discovery.Discovery{} +func NewModuleManager(ctx context.Context, fs filesystem.Filesystem) ModuleManager { + mm := newModuleManager(fs) - defaultSize := 3 * runtime.NumCPU() - wp := workerpool.New(defaultSize) + ctx, cancelFunc := context.WithCancel(ctx) + mm.cancelFunc = cancelFunc + mm.loader.Start(ctx) - mm := &moduleManager{ - modules: make([]*module, 0), - filesystem: fs, - workerPool: wp, - logger: defaultLogger, - tfDiscoFunc: d.LookPath, - tfNewExecutor: exec.NewExecutor, - } - mm.newModule = mm.defaultModuleFactory return mm } -func (mm *moduleManager) WorkerPoolSize() int { - return mm.workerPool.Size() -} - -func (mm *moduleManager) WorkerQueueSize() int { - return mm.workerPool.WaitingQueueSize() -} - -func (mm *moduleManager) defaultModuleFactory(ctx context.Context, dir string) (*module, error) { - mod := newModule(mm.filesystem, dir) +func NewSyncModuleManager(ctx context.Context, fs filesystem.Filesystem) ModuleManager { + mm := newModuleManager(fs) - mod.SetLogger(mm.logger) + ctx, cancelFunc := context.WithCancel(ctx) + mm.cancelFunc = cancelFunc + mm.syncLoading = true - d := &discovery.Discovery{} - mod.tfDiscoFunc = d.LookPath - mod.tfNewExecutor = exec.NewExecutor + mm.loader.Start(ctx) - mod.tfExecPath = mm.tfExecPath - mod.tfExecTimeout = mm.tfExecTimeout - mod.tfExecLogPath = mm.tfExecLogPath - - return mod, mod.discoverCaches(ctx, dir) -} - -func (mm *moduleManager) SetTerraformExecPath(path string) { - mm.tfExecPath = path -} - -func (mm *moduleManager) SetTerraformExecLogPath(logPath string) { - mm.tfExecLogPath = logPath + return mm } -func (mm *moduleManager) SetTerraformExecTimeout(timeout time.Duration) { - mm.tfExecTimeout = timeout +func newModuleManager(fs filesystem.Filesystem) *moduleManager { + mm := &moduleManager{ + modules: make([]*module, 0), + fs: fs, + logger: defaultLogger, + loader: newModuleLoader(), + } + return mm } func (mm *moduleManager) SetLogger(logger *log.Logger) { mm.logger = logger + mm.loader.SetLogger(logger) } -func (mm *moduleManager) InitAndUpdateModule(ctx context.Context, dir string) (Module, error) { - mod, err := mm.ModuleByPath(dir) - if err != nil { - return nil, fmt.Errorf("failed to get module: %+v", err) - } - - if err := mod.ExecuteTerraformInit(ctx); err != nil { - return nil, fmt.Errorf("failed to init module: %+v", err) - } - - m := mod.(*module) - m.discoverCaches(ctx, dir) - return mod, m.UpdateProviderSchemaCache(ctx, m.pluginLockFile) -} - -func (mm *moduleManager) AddAndStartLoadingModule(ctx context.Context, dir string) (Module, error) { - dir = filepath.Clean(dir) +func (mm *moduleManager) AddModule(modPath string) (Module, error) { + modPath = filepath.Clean(modPath) + mm.logger.Printf("MM: adding new module: %s", modPath) // TODO: Follow symlinks (requires proper test data) - if _, ok := mm.moduleByPath(dir); ok { - return nil, fmt.Errorf("module %s was already added", dir) + if _, ok := mm.moduleByPath(modPath); ok { + return nil, fmt.Errorf("module %s was already added", modPath) } - mod, err := mm.newModule(context.Background(), dir) - if err != nil { - return nil, err - } + mod := newModule(mm.fs, modPath) + mod.SetLogger(mm.logger) mm.modules = append(mm.modules, mod) - if mm.syncLoading { - mm.logger.Printf("synchronously loading module %s", dir) - return mod, mod.load(ctx) - } - - mm.logger.Printf("asynchronously loading module %s", dir) - mm.workerPool.Submit(func() { - mod := mod - err := mod.load(context.Background()) - mod.setLoadErr(err) - }) - return mod, nil } -func (mm *moduleManager) SchemaForPath(path string) (*schema.BodySchema, error) { - candidates := mm.ModuleCandidatesByPath(path) - for _, mod := range candidates { - schema, err := mod.MergedSchema() - if err != nil { - mm.logger.Printf("failed to merge schema for %s: %s", mod.Path(), err) - continue - } - if schema != nil { - mm.logger.Printf("found schema for %s at %s", path, mod.Path()) - return schema, nil - } - } - - mod, err := mm.ModuleByPath(path) +func (mm *moduleManager) EnqueueModuleOpWait(modPath string, opType OpType) error { + mod, err := mm.ModuleByPath(modPath) if err != nil { - return nil, err + return err } + modOp := NewModuleOperation(mod, opType) + mm.loader.EnqueueModuleOp(modOp) - return mod.MergedSchema() -} + <-modOp.Done() -func (mm *moduleManager) moduleByPath(dir string) (*module, bool) { - for _, mod := range mm.modules { - if pathEquals(mod.Path(), dir) { - return mod, true - } - } - return nil, false + return nil } -// ModuleCandidatesByPath finds any initialized modules -func (mm *moduleManager) ModuleCandidatesByPath(path string) Modules { - path = filepath.Clean(path) - - candidates := make([]Module, 0) - - // TODO: Follow symlinks (requires proper test data) - - mod, foundPath := mm.moduleByPath(path) - if foundPath { - inited, _ := mod.WasInitialized() - if inited { - candidates = append(candidates, mod) - } +func (mm *moduleManager) EnqueueModuleOp(modPath string, opType OpType) error { + mod, err := mm.ModuleByPath(modPath) + if err != nil { + return err } - if !foundPath { - dir := trimLockFilePath(path) - mod, ok := mm.moduleByPath(dir) - if ok { - inited, _ := mod.WasInitialized() - if inited { - candidates = append(candidates, mod) - } - } + modOp := NewModuleOperation(mod, opType) + mm.loader.EnqueueModuleOp(modOp) + if mm.syncLoading { + <-modOp.Done() } + return nil +} - for _, mod := range mm.modules { - if mod.ReferencesModulePath(path) { - candidates = append(candidates, mod) - } +func (mm *moduleManager) SchemaForModule(modPath string) (*schema.BodySchema, error) { + sources, err := mm.SchemaSourcesForModule(modPath) + if err != nil { + return nil, err } - return candidates -} + var ( + tfVersion *version.Version + coreSchema *schema.BodySchema + providerSchema *tfjson.ProviderSchemas + providerVersions map[string]*version.Version + ) -func (mm *moduleManager) ListModules() Modules { - modules := make([]Module, 0) - for _, mod := range mm.modules { - modules = append(modules, mod) + if len(sources) > 0 { + ps, err := sources[0].ProviderSchema() + if err == nil { + providerSchema = ps + } + providerVersions = sources[0].ProviderVersions() } - return modules -} -func (mm *moduleManager) ModuleByPath(path string) (Module, error) { - path = filepath.Clean(path) - if mod, ok := mm.moduleByPath(path); ok { - return mod, nil + mod, err := mm.ModuleByPath(modPath) + if err != nil { + return nil, err } - dir := trimLockFilePath(path) - - if mod, ok := mm.moduleByPath(dir); ok { - return mod, nil + if v, err := mod.TerraformVersion(); err == nil { + tfVersion = v } - return nil, &ModuleNotFoundErr{path} -} - -func (mm *moduleManager) IsProviderSchemaLoaded(path string) (bool, error) { - mod, err := mm.ModuleByPath(path) - if err != nil { - return false, err - } + if len(sources) == 0 { + mm.logger.Printf("falling back to preloaded schema for %s...", modPath) + ps, vOut, err := schemas.PreloadedProviderSchemas() + if err != nil { + return nil, err + } + if ps != nil { + providerSchema = ps + providerVersions = vOut.Providers - return mod.IsProviderSchemaLoaded(), nil -} + mm.logger.Printf("preloaded provider schema (%d providers) set for %s", + len(ps.Schemas), modPath) -func (mm *moduleManager) TerraformFormatterForDir(ctx context.Context, path string) (exec.Formatter, error) { - mod, err := mm.ModuleByPath(path) - if err != nil { - if IsModuleNotFound(err) { - return mm.newTerraformFormatter(ctx, path) + if tfVersion == nil { + tfVersion = vOut.Core + } } - return nil, err } - return mod.TerraformFormatter() -} - -func (mm *moduleManager) newTerraformFormatter(ctx context.Context, workDir string) (exec.Formatter, error) { - tfPath := mm.tfExecPath - if tfPath == "" { - var err error - tfPath, err = mm.tfDiscoFunc() + if tfVersion != nil { + coreSchema, err = tfschema.CoreModuleSchemaForVersion(tfVersion) if err != nil { return nil, err } + } else { + coreSchema = tfschema.UniversalCoreModuleSchema() } - tf, err := mm.tfNewExecutor(workDir, tfPath) - if err != nil { - return nil, err - } - - tf.SetLogger(mm.logger) - - if mm.tfExecLogPath != "" { - tf.SetExecLogPath(mm.tfExecLogPath) + merger := tfschema.NewSchemaMerger(coreSchema) + if tfVersion != nil { + merger.SetCoreVersion(tfVersion) } - - if mm.tfExecTimeout != 0 { - tf.SetTimeout(mm.tfExecTimeout) + if len(providerVersions) > 0 { + err = merger.SetProviderVersions(providerVersions) + if err != nil { + return nil, err + } } - version, _, err := tf.Version(ctx) - if err != nil { - return nil, err + pf, _ := mod.ParsedFiles() + if len(pf) > 0 { + merger.SetParsedFiles(pf) } - mm.logger.Printf("Terraform version %s found at %s (alternative)", version, tf.GetExecPath()) - return tf.Format, nil + return merger.MergeWithJsonProviderSchemas(providerSchema) } -func (mm *moduleManager) IsTerraformAvailable(path string) (bool, error) { - mod, err := mm.ModuleByPath(path) +func (mm *moduleManager) SchemaSourcesForModule(modPath string) ([]SchemaSource, error) { + mod, err := mm.ModuleByPath(modPath) if err != nil { - return false, err + return []SchemaSource{}, err } - return mod.IsTerraformAvailable(), nil -} - -func (mm *moduleManager) HasTerraformDiscoveryFinished(path string) (bool, error) { - mod, err := mm.ModuleByPath(path) - if err != nil { - return false, err + if ps, err := mod.ProviderSchema(); err == nil && ps != nil { + return []SchemaSource{mod}, nil } - return mod.HasTerraformDiscoveryFinished(), nil -} - -func (mm *moduleManager) CancelLoading() { + sources := make([]SchemaSource, 0) for _, mod := range mm.modules { - mm.logger.Printf("cancelling loading for %s", mod.Path()) - mod.CancelLoading() - mm.logger.Printf("loading cancelled for %s", mod.Path()) - } - mm.workerPool.Stop() -} - -// trimLockFilePath strips known lock file paths and filenames -// to get the directory path of the relevant module -func trimLockFilePath(filePath string) string { - pluginLockFileSuffixes := pluginLockFilePaths(string(os.PathSeparator)) - for _, s := range pluginLockFileSuffixes { - if strings.HasSuffix(filePath, s) { - return strings.TrimSuffix(filePath, s) + if mod.CallsModule(modPath) { + if ps, err := mod.ProviderSchema(); err == nil && ps != nil { + sources = append(sources, mod) + } } } - moduleManifestSuffix := moduleManifestFilePath(string(os.PathSeparator)) - if strings.HasSuffix(filePath, moduleManifestSuffix) { - return strings.TrimSuffix(filePath, moduleManifestSuffix) - } + // We could expose preloaded schemas here already + // but other logic elsewhere isn't able to take advantage + // of multiple sources and mix-and-match yet. + // TODO https://github.com/hashicorp/terraform-ls/issues/354 - return filePath + return sources, nil } -func (mm *moduleManager) PathsToWatch() []string { - paths := make([]string, 0) +func (mm *moduleManager) moduleByPath(dir string) (*module, bool) { for _, mod := range mm.modules { - ptw := mod.PathsToWatch() - if len(ptw) > 0 { - paths = append(paths, ptw...) + if pathEquals(mod.Path(), dir) { + return mod, true } } - return paths + return nil, false } -// NewModuleLoader allows adding & loading modules -// with a given context. This can be passed down to any handler -// which itself will have short-lived context -// therefore couldn't finish loading the module asynchronously -// after it responds to the client -func NewModuleLoader(ctx context.Context, mm ModuleManager) ModuleLoader { - return func(dir string) (Module, error) { - return mm.AddAndStartLoadingModule(ctx, dir) +func (mm *moduleManager) ListModules() []Module { + modules := make([]Module, 0) + for _, mod := range mm.modules { + modules = append(modules, mod) } + return modules +} +func (mm *moduleManager) ModuleByPath(path string) (Module, error) { + path = filepath.Clean(path) + + if mod, ok := mm.moduleByPath(path); ok { + return mod, nil + } + + return nil, &ModuleNotFoundErr{path} +} + +func (mm *moduleManager) CancelLoading() { + mm.cancelFunc() } diff --git a/internal/terraform/module/module_manager_mock.go b/internal/terraform/module/module_manager_mock.go index ceeff6ec7..cbcbdba6c 100644 --- a/internal/terraform/module/module_manager_mock.go +++ b/internal/terraform/module/module_manager_mock.go @@ -2,62 +2,78 @@ package module import ( "context" - "fmt" "log" + "github.com/hashicorp/go-version" + "github.com/hashicorp/terraform-json" "github.com/hashicorp/terraform-ls/internal/filesystem" - "github.com/hashicorp/terraform-ls/internal/terraform/discovery" "github.com/hashicorp/terraform-ls/internal/terraform/exec" + "github.com/stretchr/testify/mock" ) -type ModuleMockFactory struct { - mmocks map[string]*ModuleMock - logger *log.Logger - fs filesystem.Filesystem -} - -func (mmf *ModuleMockFactory) New(ctx context.Context, dir string) (*module, error) { - mmocks, ok := mmf.mmocks[dir] - if !ok { - return nil, fmt.Errorf("unexpected module requested: %s (%d available: %#v)", dir, len(mmf.mmocks), mmf.mmocks) - } - - mock := NewModuleMock(mmocks, mmf.fs, dir) - mock.SetLogger(mmf.logger) - return mock, mock.discoverCaches(ctx, dir) -} - type ModuleManagerMockInput struct { - Modules map[string]*ModuleMock - TfExecutorFactory exec.ExecutorFactory + Logger *log.Logger + TerraformCalls *exec.TerraformMockCalls } func NewModuleManagerMock(input *ModuleManagerMockInput) ModuleManagerFactory { - return func(fs filesystem.Filesystem) ModuleManager { - mm := newModuleManager(fs) - mm.syncLoading = true - - mmf := &ModuleMockFactory{ - mmocks: make(map[string]*ModuleMock, 0), - logger: mm.logger, - fs: fs, - } + var logger *log.Logger + var tfCalls *exec.TerraformMockCalls - // mock terraform discovery - md := &discovery.MockDiscovery{Path: "tf-mock"} - mm.tfDiscoFunc = md.LookPath - - // mock terraform executor - if input != nil { - mm.tfNewExecutor = input.TfExecutorFactory + if input != nil { + logger = input.Logger + tfCalls = input.TerraformCalls + } - if input.Modules != nil { - mmf.mmocks = input.Modules - } + return func(ctx context.Context, fs filesystem.Filesystem) ModuleManager { + if tfCalls != nil { + ctx = exec.WithExecutorFactory(ctx, exec.NewMockExecutor(tfCalls)) + ctx = exec.WithExecutorOpts(ctx, &exec.ExecutorOpts{ + ExecPath: "tf-mock", + }) } - mm.newModule = mmf.New + mm := NewSyncModuleManager(ctx, fs) + + if logger != nil { + mm.SetLogger(logger) + } return mm } } + +func validTfMockCalls(repeatability int) []*mock.Call { + return []*mock.Call{ + { + Method: "Version", + Repeatability: repeatability, + Arguments: []interface{}{ + mock.AnythingOfType("*context.cancelCtx"), + }, + ReturnArguments: []interface{}{ + version.Must(version.NewVersion("0.12.0")), + nil, + nil, + }, + }, + { + Method: "GetExecPath", + Repeatability: repeatability, + ReturnArguments: []interface{}{ + "", + }, + }, + { + Method: "ProviderSchemas", + Repeatability: repeatability, + Arguments: []interface{}{ + mock.AnythingOfType("*context.cancelCtx"), + }, + ReturnArguments: []interface{}{ + &tfjson.ProviderSchemas{FormatVersion: "0.1"}, + nil, + }, + }, + } +} diff --git a/internal/terraform/module/module_manager_mock_test.go b/internal/terraform/module/module_manager_mock_test.go deleted file mode 100644 index 8dbbdc712..000000000 --- a/internal/terraform/module/module_manager_mock_test.go +++ /dev/null @@ -1,74 +0,0 @@ -package module - -import ( - "context" - "os" - "path/filepath" - "testing" - - "github.com/hashicorp/go-version" - tfjson "github.com/hashicorp/terraform-json" - "github.com/hashicorp/terraform-ls/internal/filesystem" - "github.com/hashicorp/terraform-ls/internal/terraform/exec" - "github.com/stretchr/testify/mock" -) - -func TestNewModuleManagerMock_noMocks(t *testing.T) { - f := NewModuleManagerMock(nil) - mm := f(filesystem.NewFilesystem()) - _, err := mm.AddAndStartLoadingModule(context.Background(), "any-path") - if err == nil { - t.Fatal("expected unmocked path addition to fail") - } -} - -func TestNewModuleManagerMock_mocks(t *testing.T) { - tmpDir := filepath.Clean(os.TempDir()) - - f := NewModuleManagerMock(&ModuleManagerMockInput{ - Modules: map[string]*ModuleMock{ - tmpDir: { - TfExecFactory: validTfMockCalls(t, tmpDir), - }, - }}) - mm := f(filesystem.NewFilesystem()) - _, err := mm.AddAndStartLoadingModule(context.Background(), tmpDir) - if err != nil { - t.Fatal(err) - } -} - -func validTfMockCalls(t *testing.T, workDir string) exec.ExecutorFactory { - return exec.NewMockExecutor([]*mock.Call{ - { - Method: "Version", - Repeatability: 1, - Arguments: []interface{}{ - mock.AnythingOfType("*context.emptyCtx"), - }, - ReturnArguments: []interface{}{ - version.Must(version.NewVersion("0.12.0")), - nil, - nil, - }, - }, - { - Method: "GetExecPath", - Repeatability: 1, - ReturnArguments: []interface{}{ - "", - }, - }, - { - Method: "ProviderSchemas", - Repeatability: 1, - Arguments: []interface{}{ - mock.AnythingOfType("*context.emptyCtx"), - }, - ReturnArguments: []interface{}{ - &tfjson.ProviderSchemas{FormatVersion: "0.1"}, - nil, - }, - }, - }) -} diff --git a/internal/terraform/module/module_manager_test.go b/internal/terraform/module/module_manager_test.go index a8ee59097..eb3fcb13c 100644 --- a/internal/terraform/module/module_manager_test.go +++ b/internal/terraform/module/module_manager_test.go @@ -10,12 +10,8 @@ import ( "testing" "github.com/google/go-cmp/cmp" - "github.com/hashicorp/go-version" - tfjson "github.com/hashicorp/terraform-json" "github.com/hashicorp/terraform-ls/internal/filesystem" - "github.com/hashicorp/terraform-ls/internal/terraform/discovery" "github.com/hashicorp/terraform-ls/internal/terraform/exec" - "github.com/stretchr/testify/mock" ) func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { @@ -27,58 +23,33 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { testCases := []struct { name string walkerRoot string + totalModuleCount int lookupPath string expectedCandidates []string }{ - { - // outside of watcher, modules are always looked up by dir - "tf-file-based lookup", - filepath.Join(testData, "single-root-ext-modules-only"), - filepath.Join(testData, "single-root-ext-modules-only", "main.tf"), - []string{}, - }, { "dir-based lookup (exact match)", filepath.Join(testData, "single-root-ext-modules-only"), + 1, filepath.Join(testData, "single-root-ext-modules-only"), []string{ filepath.Join(testData, "single-root-ext-modules-only"), }, }, - { - "lock-file-based lookup", - filepath.Join(testData, "single-root-ext-modules-only"), - filepath.Join(testData, "single-root-ext-modules-only", - ".terraform", - "modules", - "modules.json"), - []string{ - filepath.Join(testData, "single-root-ext-modules-only"), - }, - }, { "dir-based lookup (exact match)", filepath.Join(testData, "single-root-local-and-ext-modules"), + 1, filepath.Join(testData, "single-root-local-and-ext-modules"), []string{ filepath.Join(testData, "single-root-local-and-ext-modules"), }, }, - { - "lock-file-based lookup", - filepath.Join(testData, "single-root-local-and-ext-modules"), - filepath.Join(testData, "single-root-local-and-ext-modules", - ".terraform", - "modules", - "modules.json"), - []string{ - filepath.Join(testData, "single-root-local-and-ext-modules"), - }, - }, { "mod-ref-based lookup", filepath.Join(testData, "single-root-local-and-ext-modules"), + 1, filepath.Join(testData, "single-root-local-and-ext-modules/alpha"), []string{ filepath.Join(testData, "single-root-local-and-ext-modules"), @@ -87,6 +58,7 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "mod-ref-based lookup", filepath.Join(testData, "single-root-local-and-ext-modules"), + 1, filepath.Join(testData, "single-root-local-and-ext-modules/beta"), []string{ filepath.Join(testData, "single-root-local-and-ext-modules"), @@ -95,6 +67,7 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "mod-ref-based lookup (not referenced)", filepath.Join(testData, "single-root-local-and-ext-modules"), + 1, filepath.Join(testData, "single-root-local-and-ext-modules/charlie"), []string{}, }, @@ -102,25 +75,16 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "dir-based lookup (exact match)", filepath.Join(testData, "single-root-local-modules-only"), + 1, filepath.Join(testData, "single-root-local-modules-only"), []string{ filepath.Join(testData, "single-root-local-modules-only"), }, }, - { - "lock-file-based lookup", - filepath.Join(testData, "single-root-local-modules-only"), - filepath.Join(testData, "single-root-local-modules-only", - ".terraform", - "modules", - "modules.json"), - []string{ - filepath.Join(testData, "single-root-local-modules-only"), - }, - }, { "mod-ref-based lookup", filepath.Join(testData, "single-root-local-modules-only"), + 1, filepath.Join(testData, "single-root-local-modules-only/alpha"), []string{ filepath.Join(testData, "single-root-local-modules-only"), @@ -129,6 +93,7 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "mod-ref-based lookup", filepath.Join(testData, "single-root-local-modules-only"), + 1, filepath.Join(testData, "single-root-local-modules-only/beta"), []string{ filepath.Join(testData, "single-root-local-modules-only"), @@ -137,6 +102,7 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "mod-ref-based lookup (not referenced)", filepath.Join(testData, "single-root-local-modules-only"), + 1, filepath.Join(testData, "single-root-local-modules-only/charlie"), []string{}, }, @@ -144,85 +110,46 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "dir-based lookup (exact match)", filepath.Join(testData, "single-root-no-modules"), + 1, filepath.Join(testData, "single-root-no-modules"), []string{ filepath.Join(testData, "single-root-no-modules"), }, }, - { - "lock-file-based lookup", - filepath.Join(testData, "single-root-no-modules"), - filepath.Join(testData, "single-root-no-modules", - ".terraform", - "modules", - "modules.json"), - []string{ - filepath.Join(testData, "single-root-no-modules"), - }, - }, { "directory-based lookup", filepath.Join(testData, "nested-single-root-no-modules"), + 1, filepath.Join(testData, "nested-single-root-no-modules", "tf-root"), []string{ filepath.Join(testData, "nested-single-root-no-modules", "tf-root"), }, }, - { - "lock-file-based lookup", - filepath.Join(testData, "nested-single-root-no-modules"), - filepath.Join(testData, "nested-single-root-no-modules", "tf-root", - ".terraform", - "modules", - "modules.json"), - []string{ - filepath.Join(testData, "nested-single-root-no-modules", "tf-root"), - }, - }, { "directory-based lookup", filepath.Join(testData, "nested-single-root-ext-modules-only"), + 1, filepath.Join(testData, "nested-single-root-ext-modules-only", "tf-root"), []string{ filepath.Join(testData, "nested-single-root-ext-modules-only", "tf-root"), }, }, - { - "lock-file-based lookup", - filepath.Join(testData, "nested-single-root-ext-modules-only"), - filepath.Join(testData, "nested-single-root-ext-modules-only", "tf-root", - ".terraform", - "modules", - "modules.json"), - []string{ - filepath.Join(testData, "nested-single-root-ext-modules-only", "tf-root"), - }, - }, { "directory-based lookup", filepath.Join(testData, "nested-single-root-local-modules-down"), + 1, filepath.Join(testData, "nested-single-root-local-modules-down", "tf-root"), []string{ filepath.Join(testData, "nested-single-root-local-modules-down", "tf-root"), }, }, - { - "lock-file-based lookup", - filepath.Join(testData, "nested-single-root-local-modules-down"), - filepath.Join(testData, "nested-single-root-local-modules-down", "tf-root", - ".terraform", - "modules", - "modules.json"), - []string{ - filepath.Join(testData, "nested-single-root-local-modules-down", "tf-root"), - }, - }, { "mod-based lookup", filepath.Join(testData, "nested-single-root-local-modules-down"), + 1, filepath.Join(testData, "nested-single-root-local-modules-down", "tf-root", "alpha"), []string{ filepath.Join(testData, "nested-single-root-local-modules-down", "tf-root"), @@ -231,12 +158,14 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "mod-based lookup", filepath.Join(testData, "nested-single-root-local-modules-down"), + 1, filepath.Join(testData, "nested-single-root-local-modules-down", "tf-root", "beta"), []string{}, }, { "mod-based lookup", filepath.Join(testData, "nested-single-root-local-modules-down"), + 1, filepath.Join(testData, "nested-single-root-local-modules-down", "tf-root", "charlie"), []string{ filepath.Join(testData, "nested-single-root-local-modules-down", "tf-root"), @@ -246,25 +175,16 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "dir-based lookup", filepath.Join(testData, "nested-single-root-local-modules-up"), + 1, filepath.Join(testData, "nested-single-root-local-modules-up", "module", "tf-root"), []string{ filepath.Join(testData, "nested-single-root-local-modules-up", "module", "tf-root"), }, }, - { - "lock-file-based lookup", - filepath.Join(testData, "nested-single-root-local-modules-up"), - filepath.Join(testData, "nested-single-root-local-modules-up", "module", "tf-root", - ".terraform", - "modules", - "modules.json"), - []string{ - filepath.Join(testData, "nested-single-root-local-modules-up", "module", "tf-root"), - }, - }, { "mod-based lookup", filepath.Join(testData, "nested-single-root-local-modules-up"), + 1, filepath.Join(testData, "nested-single-root-local-modules-up", "module"), []string{ filepath.Join(testData, "nested-single-root-local-modules-up", "module", "tf-root"), @@ -276,6 +196,7 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "directory-env-based lookup", filepath.Join(testData, "main-module-multienv"), + 3, filepath.Join(testData, "main-module-multienv", "env", "dev"), []string{ filepath.Join(testData, "main-module-multienv", "env", "dev"), @@ -284,6 +205,7 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "directory-env-based lookup", filepath.Join(testData, "main-module-multienv"), + 3, filepath.Join(testData, "main-module-multienv", "env", "prod"), []string{ filepath.Join(testData, "main-module-multienv", "env", "prod"), @@ -292,6 +214,7 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "main module lookup", filepath.Join(testData, "main-module-multienv"), + 3, filepath.Join(testData, "main-module-multienv", "main"), []string{ filepath.Join(testData, "main-module-multienv", "env", "dev"), @@ -303,6 +226,7 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "dir-based lookup", filepath.Join(testData, "multi-root-no-modules"), + 3, filepath.Join(testData, "multi-root-no-modules", "first-root"), []string{ filepath.Join(testData, "multi-root-no-modules", "first-root"), @@ -311,56 +235,26 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "dir-based lookup", filepath.Join(testData, "multi-root-no-modules"), + 3, filepath.Join(testData, "multi-root-no-modules", "second-root"), []string{ filepath.Join(testData, "multi-root-no-modules", "second-root"), }, }, - { - "lock-file-based lookup", - filepath.Join(testData, "multi-root-no-modules"), - filepath.Join(testData, "multi-root-no-modules", "first-root", - ".terraform", - "modules", - "modules.json"), - []string{ - filepath.Join(testData, "multi-root-no-modules", "first-root"), - }, - }, - { - "lock-file-based lookup", - filepath.Join(testData, "multi-root-no-modules"), - filepath.Join(testData, "multi-root-no-modules", "second-root", - ".terraform", - "modules", - "modules.json"), - []string{ - filepath.Join(testData, "multi-root-no-modules", "second-root"), - }, - }, { "dir-based lookup", filepath.Join(testData, "multi-root-local-modules-down"), + 3, filepath.Join(testData, "multi-root-local-modules-down", "first-root"), []string{ filepath.Join(testData, "multi-root-local-modules-down", "first-root"), }, }, - { - "lock-file-based lookup", - filepath.Join(testData, "multi-root-local-modules-down"), - filepath.Join(testData, "multi-root-local-modules-down", "first-root", - ".terraform", - "modules", - "modules.json"), - []string{ - filepath.Join(testData, "multi-root-local-modules-down", "first-root"), - }, - }, { "mod-based lookup", filepath.Join(testData, "multi-root-local-modules-down"), + 3, filepath.Join(testData, "multi-root-local-modules-down", "first-root", "alpha"), []string{ filepath.Join(testData, "multi-root-local-modules-down", "first-root"), @@ -369,12 +263,14 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "mod-based lookup", filepath.Join(testData, "multi-root-local-modules-down"), + 3, filepath.Join(testData, "multi-root-local-modules-down", "first-root", "beta"), []string{}, }, { "mod-based lookup", filepath.Join(testData, "multi-root-local-modules-down"), + 3, filepath.Join(testData, "multi-root-local-modules-down", "first-root", "charlie"), []string{ filepath.Join(testData, "multi-root-local-modules-down", "first-root"), @@ -383,25 +279,16 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "dir-based lookup", filepath.Join(testData, "multi-root-local-modules-down"), + 3, filepath.Join(testData, "multi-root-local-modules-down", "second-root"), []string{ filepath.Join(testData, "multi-root-local-modules-down", "second-root"), }, }, - { - "lock-file-based lookup", - filepath.Join(testData, "multi-root-local-modules-down"), - filepath.Join(testData, "multi-root-local-modules-down", "second-root", - ".terraform", - "modules", - "modules.json"), - []string{ - filepath.Join(testData, "multi-root-local-modules-down", "second-root"), - }, - }, { "mod-based lookup", filepath.Join(testData, "multi-root-local-modules-down"), + 3, filepath.Join(testData, "multi-root-local-modules-down", "second-root", "alpha"), []string{ filepath.Join(testData, "multi-root-local-modules-down", "second-root"), @@ -410,12 +297,14 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "mod-based lookup", filepath.Join(testData, "multi-root-local-modules-down"), + 3, filepath.Join(testData, "multi-root-local-modules-down", "second-root", "beta"), []string{}, }, { "mod-based lookup", filepath.Join(testData, "multi-root-local-modules-down"), + 3, filepath.Join(testData, "multi-root-local-modules-down", "second-root", "charlie"), []string{ filepath.Join(testData, "multi-root-local-modules-down", "second-root"), @@ -425,6 +314,7 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "dir-based lookup", filepath.Join(testData, "multi-root-local-modules-up"), + 3, filepath.Join(testData, "multi-root-local-modules-up", "main-module"), []string{ filepath.Join(testData, "multi-root-local-modules-up", "main-module", "modules", "first"), @@ -437,28 +327,57 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { for i, tc := range testCases { base := filepath.Base(tc.walkerRoot) t.Run(fmt.Sprintf("%d-%s/%s", i, tc.name, base), func(t *testing.T) { - mm := testModuleManager(t) - w := MockWalker() - w.SetLogger(testLogger()) ctx := context.Background() - err := w.StartWalking(ctx, tc.walkerRoot, func(ctx context.Context, modPath string) error { - _, err := mm.AddAndStartLoadingModule(ctx, modPath) - return err + fs := filesystem.NewFilesystem() + mmock := NewModuleManagerMock(&ModuleManagerMockInput{ + Logger: testLogger(), + TerraformCalls: &exec.TerraformMockCalls{ + AnyWorkDir: validTfMockCalls(tc.totalModuleCount), + }, }) + mm := mmock(ctx, fs) + t.Cleanup(mm.CancelLoading) + + w := SyncWalker(fs, mm) + w.SetLogger(testLogger()) + err := w.StartWalking(ctx, tc.walkerRoot) if err != nil { t.Fatal(err) } - candidates := mm.ModuleCandidatesByPath(tc.lookupPath) - if diff := cmp.Diff(tc.expectedCandidates, candidates.Paths()); diff != "" { + mm.AddModule(tc.lookupPath) + + candidates, err := mm.SchemaSourcesForModule(tc.lookupPath) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(tc.expectedCandidates, schemaSourcesPaths(t, candidates)); diff != "" { t.Fatalf("candidates don't match: %s", diff) } }) } } -func TestSchemaForPath_uninitialized(t *testing.T) { - mm := testModuleManager(t) +func schemaSourcesPaths(t *testing.T, srcs []SchemaSource) []string { + paths := make([]string, len(srcs)) + for i, src := range srcs { + mod, ok := src.(Module) + if !ok { + t.Fatal("schema source is not Module compatible") + } + paths[i] = mod.Path() + } + + return paths +} + +func TestSchemaForModule_uninitialized(t *testing.T) { + mmock := NewModuleManagerMock(nil) + + ctx := context.Background() + fs := filesystem.NewFilesystem() + mm := mmock(ctx, fs) + t.Cleanup(mm.CancelLoading) testData, err := filepath.Abs("testdata") if err != nil { @@ -466,80 +385,17 @@ func TestSchemaForPath_uninitialized(t *testing.T) { } path := filepath.Join(testData, "uninitialized-root") - // model is added automatically during didOpen - _, err = mm.AddAndStartLoadingModule(context.Background(), path) + _, err = mm.AddModule(path) if err != nil { t.Fatal(err) } - _, err = mm.SchemaForPath(path) + _, err = mm.SchemaForModule(path) if err != nil { t.Fatal(err) } } -func testModuleManager(t *testing.T) *moduleManager { - fs := filesystem.NewFilesystem() - mm := newModuleManager(fs) - mm.syncLoading = true - mm.logger = testLogger() - - mm.newModule = func(ctx context.Context, dir string) (*module, error) { - // TODO(RS): Should be just 1, unsure why it requires 2 - repeatability := 2 - mod := NewModuleMock(&ModuleMock{ - TfExecFactory: exec.NewMockExecutor([]*mock.Call{ - { - Method: "Version", - Repeatability: repeatability, - Arguments: []interface{}{ - mock.AnythingOfType(""), - }, - ReturnArguments: []interface{}{ - version.Must(version.NewVersion("0.12.0")), - nil, - nil, - }, - }, - { - Method: "GetExecPath", - Repeatability: repeatability, - ReturnArguments: []interface{}{ - "", - }, - }, - { - Method: "ProviderSchemas", - Repeatability: repeatability, - Arguments: []interface{}{ - mock.AnythingOfType(""), - }, - ReturnArguments: []interface{}{ - &tfjson.ProviderSchemas{FormatVersion: "0.1"}, - nil, - }, - }, - }), - }, fs, dir) - mod.logger = testLogger() - md := &discovery.MockDiscovery{Path: "tf-mock"} - mod.tfDiscoFunc = md.LookPath - - err := mod.discoverCaches(ctx, dir) - if err != nil { - t.Fatal(err) - } - - err = mod.load(ctx) - if err != nil { - t.Fatal(err) - } - - return mod, nil - } - return mm -} - func testLogger() *log.Logger { if testing.Verbose() { return log.New(os.Stdout, "", log.LstdFlags|log.Lshortfile) diff --git a/internal/terraform/module/module_manifest_test.go b/internal/terraform/module/module_manifest_test.go deleted file mode 100644 index 84a6c51ec..000000000 --- a/internal/terraform/module/module_manifest_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package module - -const moduleManifestRecord_external = `{ - "Key": "web_server_sg", - "Source": "terraform-aws-modules/security-group/aws//modules/http-80", - "Version": "3.10.0", - "Dir": ".terraform/modules/web_server_sg/terraform-aws-security-group-3.10.0/modules/http-80" -}` - -const moduleManifestRecord_externalDirtyPath = `{ - "Key": "web_server_sg", - "Source": "terraform-aws-modules/security-group/aws//modules/http-80", - "Version": "3.10.0", - "Dir": ".terraform/modules/web_server_sg/terraform-aws-security-group-3.10.0/modules/something/../http-80" -}` - -const moduleManifestRecord_local = `{ - "Key": "local", - "Source": "./nested/path", - "Dir": "nested/path" -}` - -const moduleManifestRecord_root = `{ - "Key": "", - "Source": "", - "Dir": "." -}` diff --git a/internal/terraform/module/module_mock.go b/internal/terraform/module/module_mock.go deleted file mode 100644 index 0c24a7e16..000000000 --- a/internal/terraform/module/module_mock.go +++ /dev/null @@ -1,30 +0,0 @@ -package module - -import ( - tfjson "github.com/hashicorp/terraform-json" - "github.com/hashicorp/terraform-ls/internal/filesystem" - "github.com/hashicorp/terraform-ls/internal/terraform/discovery" - "github.com/hashicorp/terraform-ls/internal/terraform/exec" -) - -type ModuleMock struct { - TfExecFactory exec.ExecutorFactory - ProviderSchemas *tfjson.ProviderSchemas -} - -func NewModuleMock(modMock *ModuleMock, fs filesystem.Filesystem, dir string) *module { - module := newModule(fs, dir) - - // mock terraform discovery - md := &discovery.MockDiscovery{Path: "tf-mock"} - module.tfDiscoFunc = md.LookPath - - // mock terraform executor - module.tfNewExecutor = modMock.TfExecFactory - - if modMock.ProviderSchemas != nil { - module.providerSchema = modMock.ProviderSchemas - } - - return module -} diff --git a/internal/terraform/module/module_ops.go b/internal/terraform/module/module_ops.go new file mode 100644 index 000000000..114a9b70d --- /dev/null +++ b/internal/terraform/module/module_ops.go @@ -0,0 +1,214 @@ +package module + +import ( + "context" + "fmt" + "path/filepath" + "strings" + + "github.com/hashicorp/hcl-lang/decoder" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/hashicorp/terraform-ls/internal/terraform/datadir" +) + +type OpState uint + +const ( + OpStateUnknown OpState = iota + OpStateQueued + OpStateLoading + OpStateLoaded +) + +type OpType uint + +const ( + OpTypeUnknown OpType = iota + OpTypeGetTerraformVersion + OpTypeObtainSchema + OpTypeParseConfiguration + OpTypeParseModuleManifest +) + +func (t OpType) String() string { + switch t { + case OpTypeUnknown: + return "OpTypeUnknown" + case OpTypeGetTerraformVersion: + return "OpTypeGetTerraformVersion" + case OpTypeObtainSchema: + return "OpTypeObtainSchema" + case OpTypeParseConfiguration: + return "OpTypeParseConfiguration" + case OpTypeParseModuleManifest: + return "OpTypeParseModuleManifest" + } + + return fmt.Sprintf("OpType(%d)", t) +} + +type ModuleOperation struct { + Module Module + Type OpType + + doneCh chan struct{} +} + +func NewModuleOperation(mod Module, typ OpType) ModuleOperation { + return ModuleOperation{ + Module: mod, + Type: typ, + doneCh: make(chan struct{}, 1), + } +} + +func (mo ModuleOperation) markAsDone() { + mo.doneCh <- struct{}{} +} + +func (mo ModuleOperation) Done() <-chan struct{} { + return mo.doneCh +} + +func GetTerraformVersion(ctx context.Context, mod Module) { + m := mod.(*module) + + m.SetTerraformVersionState(OpStateLoading) + defer m.SetTerraformVersionState(OpStateLoaded) + + tfExec, err := TerraformExecutorForModule(ctx, mod) + if err != nil { + m.SetTerraformVersion(nil, err) + m.logger.Printf("getting executor failed: %s", err) + return + } + + v, pv, err := tfExec.Version(ctx) + if err != nil { + m.logger.Printf("failed to get terraform version: %s", err) + } else { + m.logger.Printf("got terraform version successfully for %s", m.Path()) + } + + m.SetTerraformVersion(v, err) + if len(pv) > 0 { + m.SetProviderVersions(pv) + } +} + +func ObtainSchema(ctx context.Context, mod Module) { + m := mod.(*module) + m.SetProviderSchemaObtainingState(OpStateLoading) + defer m.SetProviderSchemaObtainingState(OpStateLoaded) + + tfExec, err := TerraformExecutorForModule(ctx, mod) + if err != nil { + m.SetProviderSchemas(nil, err) + m.logger.Printf("getting executor failed: %s", err) + return + } + + ps, err := tfExec.ProviderSchemas(ctx) + if err != nil { + m.logger.Printf("failed to obtain schema: %s", err) + } else { + m.logger.Printf("schema obtained successfully for %s", m.Path()) + } + + m.SetProviderSchemas(ps, err) +} + +func ParseConfiguration(mod Module) { + m := mod.(*module) + m.SetConfigParsingState(OpStateLoading) + defer m.SetConfigParsingState(OpStateLoaded) + + files := make(map[string]*hcl.File, 0) + diags := make(map[string]hcl.Diagnostics, 0) + + infos, err := m.fs.ReadDir(m.Path()) + if err != nil { + m.SetParsedFiles(files, err) + return + } + + for _, info := range infos { + if info.IsDir() { + // We only care about files + continue + } + + name := info.Name() + if !strings.HasSuffix(name, ".tf") || IsIgnoredFile(name) { + continue + } + + // TODO: overrides + + fullPath := filepath.Join(m.Path(), name) + + src, err := m.fs.ReadFile(fullPath) + if err != nil { + m.SetParsedFiles(files, err) + return + } + + f, pDiags := hclsyntax.ParseConfig(src, name, hcl.InitialPos) + diags[name] = pDiags + if f != nil { + files[name] = f + } + } + + m.SetParsedFiles(files, err) + m.SetDiagnostics(diags) + return +} + +// IsIgnoredFile returns true if the given filename (which must not have a +// directory path ahead of it) should be ignored as e.g. an editor swap file. +func IsIgnoredFile(name string) bool { + return strings.HasPrefix(name, ".") || // Unix-like hidden files + strings.HasSuffix(name, "~") || // vim + strings.HasPrefix(name, "#") && strings.HasSuffix(name, "#") // emacs +} + +func ParseModuleManifest(mod Module) { + m := mod.(*module) + m.SetModuleManifestParsingState(OpStateLoading) + defer m.SetModuleManifestParsingState(OpStateLoaded) + + manifestPath, ok := datadir.ModuleManifestFilePath(m.fs, mod.Path()) + if !ok { + m.logger.Printf("%s: manifest file does not exist", mod.Path()) + return + } + + mm, err := datadir.ParseModuleManifestFromFile(manifestPath) + if err != nil { + m.logger.Printf("failed to parse manifest: %s", err) + } else { + m.logger.Printf("manifest parsed successfully for %s", m.Path()) + } + + m.SetModuleManifest(mm, err) +} + +func DecoderForModule(mod Module) (*decoder.Decoder, error) { + d := decoder.NewDecoder() + + pf, err := mod.ParsedFiles() + if err != nil { + return nil, err + } + + for name, f := range pf { + err := d.LoadFile(name, f) + if err != nil { + return nil, fmt.Errorf("failed to load a file: %w", err) + } + } + + return d, nil +} diff --git a/internal/terraform/module/module_ops_queue.go b/internal/terraform/module/module_ops_queue.go new file mode 100644 index 000000000..f4d0677ab --- /dev/null +++ b/internal/terraform/module/module_ops_queue.go @@ -0,0 +1,99 @@ +package module + +import ( + "container/heap" + "sync" +) + +type moduleOpsQueue struct { + q queue + mu *sync.Mutex +} + +func newModuleOpsQueue() moduleOpsQueue { + q := moduleOpsQueue{ + q: make(queue, 0), + mu: &sync.Mutex{}, + } + heap.Init(&q.q) + return q +} + +func (q *moduleOpsQueue) PushOp(op ModuleOperation) { + q.mu.Lock() + defer q.mu.Unlock() + + heap.Push(&q.q, op) + +} + +func (q *moduleOpsQueue) PopOp() ModuleOperation { + q.mu.Lock() + defer q.mu.Unlock() + + item := heap.Pop(&q.q) + modOp := item.(ModuleOperation) + return modOp +} + +func (q *moduleOpsQueue) Len() int { + q.mu.Lock() + defer q.mu.Unlock() + + return q.q.Len() +} + +func (q *moduleOpsQueue) Peek() interface{} { + q.mu.Lock() + defer q.mu.Unlock() + + item := q.q.Peek() + return item +} + +type queue []ModuleOperation + +var _ heap.Interface = &queue{} + +func (q *queue) Push(x interface{}) { + modOp := x.(ModuleOperation) + *q = append(*q, modOp) +} + +func (q queue) Swap(i, j int) { + q[i], q[j] = q[j], q[i] +} + +func (q *queue) Pop() interface{} { + old := *q + n := len(old) + item := old[n-1] + *q = old[0 : n-1] + return item +} + +func (q queue) Peek() interface{} { + n := len(q) + return q[n-1] +} + +func (q queue) Len() int { + return len(q) +} + +func (q queue) Less(i, j int) bool { + return moduleOperationLess(q[i], q[j]) +} + +func moduleOperationLess(aModOp, bModOp ModuleOperation) bool { + leftOpen, rightOpen := 0, 0 + + if aModOp.Module.HasOpenFiles() { + leftOpen = 1 + } + if bModOp.Module.HasOpenFiles() { + rightOpen = 1 + } + + return leftOpen > rightOpen +} diff --git a/internal/terraform/module/module_ops_queue_test.go b/internal/terraform/module/module_ops_queue_test.go new file mode 100644 index 000000000..d43c06b7d --- /dev/null +++ b/internal/terraform/module/module_ops_queue_test.go @@ -0,0 +1,91 @@ +package module + +import ( + "os" + "path/filepath" + "testing" + + "github.com/hashicorp/terraform-ls/internal/filesystem" + ilsp "github.com/hashicorp/terraform-ls/internal/lsp" + "github.com/hashicorp/terraform-ls/internal/protocol" + "github.com/hashicorp/terraform-ls/internal/uri" +) + +func TestModuleOpsQueue_modulePriority(t *testing.T) { + mq := newModuleOpsQueue() + + fs := filesystem.NewFilesystem() + fs.SetLogger(testLogger()) + + dir := t.TempDir() + t.Cleanup(func() { + os.RemoveAll(dir) + }) + + ops := []ModuleOperation{ + NewModuleOperation( + closedModAtPath(t, fs, dir, "alpha"), + OpTypeGetTerraformVersion, + ), + NewModuleOperation( + openModAtPath(t, fs, dir, "beta"), + OpTypeGetTerraformVersion, + ), + NewModuleOperation( + openModAtPath(t, fs, dir, "gamma"), + OpTypeGetTerraformVersion, + ), + NewModuleOperation( + closedModAtPath(t, fs, dir, "delta"), + OpTypeGetTerraformVersion, + ), + } + + for _, op := range ops { + mq.PushOp(op) + } + + firstOp := mq.PopOp() + + expectedFirstPath := filepath.Join(dir, "beta") + firstPath := firstOp.Module.Path() + if firstPath != expectedFirstPath { + t.Fatalf("path mismatch\nexpected: %s\ngiven: %s", + expectedFirstPath, firstPath) + } + + secondOp := mq.PopOp() + expectedSecondPath := filepath.Join(dir, "gamma") + secondPath := secondOp.Module.Path() + if secondPath != expectedSecondPath { + t.Fatalf("path mismatch\nexpected: %s\ngiven: %s", + expectedSecondPath, secondPath) + } +} + +func closedModAtPath(t *testing.T, fs filesystem.Filesystem, dir, modName string) Module { + modPath := filepath.Join(dir, modName) + + docPath := filepath.Join(modPath, "main.tf") + dh := ilsp.FileHandlerFromDocumentURI(protocol.DocumentURI(uri.FromPath(docPath))) + err := fs.CreateDocument(dh, []byte{}) + if err != nil { + t.Fatal(err) + } + m := newModule(fs, modPath) + m.SetLogger(testLogger()) + return m +} + +func openModAtPath(t *testing.T, fs filesystem.Filesystem, dir, modName string) Module { + modPath := filepath.Join(dir, modName) + docPath := filepath.Join(modPath, "main.tf") + dh := ilsp.FileHandlerFromDocumentURI(protocol.DocumentURI(uri.FromPath(docPath))) + err := fs.CreateAndOpenDocument(dh, []byte{}) + if err != nil { + t.Fatal(err) + } + m := newModule(fs, modPath) + m.SetLogger(testLogger()) + return m +} diff --git a/internal/terraform/module/path.go b/internal/terraform/module/path.go index 035391a18..aa3f45b37 100644 --- a/internal/terraform/module/path.go +++ b/internal/terraform/module/path.go @@ -6,6 +6,8 @@ import ( ) func pathEquals(path1, path2 string) bool { + path1 = filepath.Clean(path1) + path2 = filepath.Clean(path2) volume1 := filepath.VolumeName(path1) volume2 := filepath.VolumeName(path2) return strings.EqualFold(volume1, volume2) && path1[len(volume1):] == path2[len(volume2):] diff --git a/internal/terraform/module/plugin_lock_file.go b/internal/terraform/module/plugin_lock_file.go deleted file mode 100644 index b160d5733..000000000 --- a/internal/terraform/module/plugin_lock_file.go +++ /dev/null @@ -1,26 +0,0 @@ -package module - -import ( - "path/filepath" - "runtime" -) - -func pluginLockFilePaths(dir string) []string { - return []string{ - // Terraform >= 0.14 - filepath.Join(dir, - ".terraform.lock.hcl", - ), - // Terraform >= v0.13 - filepath.Join(dir, - ".terraform", - "plugins", - "selections.json"), - // Terraform <= v0.12 - filepath.Join(dir, - ".terraform", - "plugins", - runtime.GOOS+"_"+runtime.GOARCH, - "lock.json"), - } -} diff --git a/internal/terraform/module/terraform_executor.go b/internal/terraform/module/terraform_executor.go new file mode 100644 index 000000000..87d4b9aaf --- /dev/null +++ b/internal/terraform/module/terraform_executor.go @@ -0,0 +1,42 @@ +package module + +import ( + "context" + "fmt" + + "github.com/hashicorp/terraform-ls/internal/terraform/exec" +) + +func TerraformExecutorForModule(ctx context.Context, mod Module) (exec.TerraformExecutor, error) { + newExecutor, ok := exec.ExecutorFactoryFromContext(ctx) + if !ok { + return nil, fmt.Errorf("no terraform executor provided") + } + + var tfExec exec.TerraformExecutor + var err error + + opts, ok := exec.ExecutorOptsFromContext(ctx) + if ok && opts.ExecPath != "" { + tfExec, err = newExecutor(mod.Path(), opts.ExecPath) + if err != nil { + return nil, err + } + } else if mod.TerraformExecPath() != "" { + tfExec, err = newExecutor(mod.Path(), mod.TerraformExecPath()) + if err != nil { + return nil, err + } + } else { + return nil, fmt.Errorf("no exec path provided for terraform") + } + + if ok && opts.ExecLogPath != "" { + tfExec.SetExecLogPath(opts.ExecLogPath) + } + if ok && opts.Timeout != 0 { + tfExec.SetTimeout(opts.Timeout) + } + + return tfExec, nil +} diff --git a/internal/terraform/module/types.go b/internal/terraform/module/types.go index 2e1ef5561..0df0b8380 100644 --- a/internal/terraform/module/types.go +++ b/internal/terraform/module/types.go @@ -3,93 +3,78 @@ package module import ( "context" "log" - "time" - "github.com/hashicorp/hcl-lang/decoder" + "github.com/hashicorp/go-version" "github.com/hashicorp/hcl-lang/schema" "github.com/hashicorp/hcl/v2" + tfjson "github.com/hashicorp/terraform-json" "github.com/hashicorp/terraform-ls/internal/filesystem" - "github.com/hashicorp/terraform-ls/internal/terraform/exec" + "github.com/hashicorp/terraform-ls/internal/terraform/datadir" ) type File interface { Path() string } -type TerraformFormatterFinder interface { - TerraformFormatterForDir(ctx context.Context, path string) (exec.Formatter, error) - HasTerraformDiscoveryFinished(path string) (bool, error) - IsTerraformAvailable(path string) (bool, error) +type SchemaSource interface { + // module specific methods + Path() string + HumanReadablePath(string) string + + ProviderSchema() (*tfjson.ProviderSchemas, error) + TerraformVersion() (*version.Version, error) + ProviderVersions() map[string]*version.Version } type ModuleFinder interface { - ModuleCandidatesByPath(path string) Modules ModuleByPath(path string) (Module, error) - SchemaForPath(path string) (*schema.BodySchema, error) + SchemaForModule(path string) (*schema.BodySchema, error) + SchemaSourcesForModule(path string) ([]SchemaSource, error) } type ModuleLoader func(dir string) (Module, error) type ModuleManager interface { ModuleFinder - TerraformFormatterFinder SetLogger(logger *log.Logger) - - SetTerraformExecPath(path string) - SetTerraformExecLogPath(logPath string) - SetTerraformExecTimeout(timeout time.Duration) - - InitAndUpdateModule(ctx context.Context, dir string) (Module, error) - AddAndStartLoadingModule(ctx context.Context, dir string) (Module, error) - WorkerPoolSize() int - WorkerQueueSize() int - ListModules() Modules - PathsToWatch() []string + AddModule(modPath string) (Module, error) + EnqueueModuleOp(modPath string, opType OpType) error + EnqueueModuleOpWait(modPath string, opType OpType) error + ListModules() []Module CancelLoading() } -type Modules []Module - -func (mods Modules) Paths() []string { - paths := make([]string, len(mods)) - for i, mod := range mods { - paths[i] = mod.Path() - } - return paths -} - type Module interface { Path() string - MatchesPath(path string) bool - LoadError() error - StartLoading() error - IsLoadingDone() bool - LoadingDone() <-chan struct{} - IsKnownPluginLockFile(path string) bool - IsKnownModuleManifestFile(path string) bool - PathsToWatch() []string - UpdateProviderSchemaCache(ctx context.Context, lockFile File) error - IsProviderSchemaLoaded() bool - UpdateModuleManifest(manifestFile File) error - Decoder() (*decoder.Decoder, error) - DecoderWithSchema(*schema.BodySchema) (*decoder.Decoder, error) - MergedSchema() (*schema.BodySchema, error) - IsParsed() bool - ParseFiles() error - ParsedDiagnostics() map[string]hcl.Diagnostics - TerraformFormatter() (exec.Formatter, error) - HasTerraformDiscoveryFinished() bool - IsTerraformAvailable() bool - ExecuteTerraformInit(ctx context.Context) error - ExecuteTerraformValidate(ctx context.Context) (map[string]hcl.Diagnostics, error) - Modules() []ModuleRecord HumanReadablePath(string) string - WasInitialized() (bool, error) + MatchesPath(path string) bool + HasOpenFiles() bool + + TerraformExecPath() string + TerraformVersion() (*version.Version, error) + ProviderVersions() map[string]*version.Version + ProviderSchema() (*tfjson.ProviderSchemas, error) + ModuleManifest() (*datadir.ModuleManifest, error) + + TerraformVersionState() OpState + ProviderSchemaState() OpState + + ParsedFiles() (map[string]*hcl.File, error) + Diagnostics() map[string]hcl.Diagnostics + ModuleCalls() []datadir.ModuleRecord } -type ModuleFactory func(context.Context, string) (*module, error) +type ModuleFactory func(string) (*module, error) + +type ModuleManagerFactory func(context.Context, filesystem.Filesystem) ModuleManager -type ModuleManagerFactory func(filesystem.Filesystem) ModuleManager +type WalkerFactory func(filesystem.Filesystem, ModuleManager) *Walker -type WalkerFactory func() *Walker +type Watcher interface { + Start() error + Stop() error + SetLogger(*log.Logger) + AddModule(string) error + IsModuleWatched(string) bool +} diff --git a/internal/terraform/module/walker.go b/internal/terraform/module/walker.go index 172549600..66a484017 100644 --- a/internal/terraform/module/walker.go +++ b/internal/terraform/module/walker.go @@ -8,6 +8,9 @@ import ( "os" "path/filepath" "sync" + + "github.com/hashicorp/terraform-ls/internal/filesystem" + "github.com/hashicorp/terraform-ls/internal/terraform/datadir" ) var ( @@ -24,8 +27,11 @@ var ( ) type Walker struct { - logger *log.Logger - sync bool + fs filesystem.Filesystem + modMgr ModuleManager + watcher Watcher + logger *log.Logger + sync bool walking bool walkingMu *sync.RWMutex @@ -35,8 +41,10 @@ type Walker struct { excludeModulePaths map[string]bool } -func NewWalker() *Walker { +func NewWalker(fs filesystem.Filesystem, modMgr ModuleManager) *Walker { return &Walker{ + fs: fs, + modMgr: modMgr, logger: discardLogger, walkingMu: &sync.RWMutex{}, doneCh: make(chan struct{}, 0), @@ -47,6 +55,10 @@ func (w *Walker) SetLogger(logger *log.Logger) { w.logger = logger } +func (w *Walker) SetWatcher(watcher Watcher) { + w.watcher = watcher +} + func (w *Walker) SetExcludeModulePaths(excludeModulePaths []string) { w.excludeModulePaths = make(map[string]bool) for _, path := range excludeModulePaths { @@ -54,8 +66,6 @@ func (w *Walker) SetExcludeModulePaths(excludeModulePaths []string) { } } -type WalkFunc func(ctx context.Context, rootModulePath string) error - func (w *Walker) Stop() { if w.cancelFunc != nil { w.cancelFunc() @@ -73,11 +83,7 @@ func (w *Walker) setWalking(isWalking bool) { w.walking = isWalking } -func (w *Walker) Done() <-chan struct{} { - return w.doneCh -} - -func (w *Walker) StartWalking(ctx context.Context, path string, wf WalkFunc) error { +func (w *Walker) StartWalking(ctx context.Context, path string) error { if w.IsWalking() { return fmt.Errorf("walker is already running") } @@ -89,18 +95,18 @@ func (w *Walker) StartWalking(ctx context.Context, path string, wf WalkFunc) err if w.sync { w.logger.Printf("synchronously walking through %s", path) - return w.walk(ctx, path, wf) + return w.walk(ctx, path) } - go func(w *Walker, path string, wf WalkFunc) { + go func(w *Walker, path string) { w.logger.Printf("asynchronously walking through %s", path) - err := w.walk(ctx, path, wf) + err := w.walk(ctx, path) if err != nil { w.logger.Printf("async walking through %s failed: %s", path, err) return } w.logger.Printf("async walking through %s finished", path) - }(w, path, wf) + }(w, path) return nil } @@ -112,9 +118,12 @@ func (w *Walker) IsWalking() bool { return w.walking } -func (w *Walker) walk(ctx context.Context, rootPath string, wf WalkFunc) error { +func (w *Walker) walk(ctx context.Context, rootPath string) error { defer w.Stop() + // We ignore the passed FS and instead read straight from OS FS + // because that would require reimplementing filepath.Walk and + // the data directory should never be on the virtual filesystem anyway err := filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error { select { case <-w.doneCh: @@ -137,9 +146,45 @@ func (w *Walker) walk(ctx context.Context, rootPath string, wf WalkFunc) error { return filepath.SkipDir } - if info.Name() == ".terraform" { + if info.Name() == datadir.DataDirName { w.logger.Printf("found module %s", dir) - return wf(ctx, dir) + + _, err := w.modMgr.ModuleByPath(dir) + if err != nil { + if IsModuleNotFound(err) { + _, err := w.modMgr.AddModule(dir) + if err != nil { + return err + } + } else { + return err + } + } + + err = w.modMgr.EnqueueModuleOp(dir, OpTypeGetTerraformVersion) + if err != nil { + return err + } + + dataDir := datadir.WalkDataDirOfModule(w.fs, dir) + if dataDir.ModuleManifestPath != "" { + err = w.modMgr.EnqueueModuleOp(dir, OpTypeParseModuleManifest) + if err != nil { + return err + } + } + if dataDir.PluginLockFilePath != "" { + err = w.modMgr.EnqueueModuleOp(dir, OpTypeObtainSchema) + if err != nil { + return err + } + } + + if w.watcher != nil { + w.watcher.AddModule(dir) + } + + return nil } if !info.IsDir() { diff --git a/internal/terraform/module/walker_mock.go b/internal/terraform/module/walker_mock.go index 1c4b55798..bebdb4536 100644 --- a/internal/terraform/module/walker_mock.go +++ b/internal/terraform/module/walker_mock.go @@ -1,7 +1,9 @@ package module -func MockWalker() *Walker { - w := NewWalker() +import "github.com/hashicorp/terraform-ls/internal/filesystem" + +func SyncWalker(fs filesystem.Filesystem, modMgr ModuleManager) *Walker { + w := NewWalker(fs, modMgr) w.sync = true return w } diff --git a/internal/terraform/module/watcher.go b/internal/terraform/module/watcher.go new file mode 100644 index 000000000..b41354c27 --- /dev/null +++ b/internal/terraform/module/watcher.go @@ -0,0 +1,234 @@ +package module + +import ( + "context" + "io/ioutil" + "log" + "os" + "path/filepath" + + "github.com/fsnotify/fsnotify" + "github.com/hashicorp/terraform-ls/internal/filesystem" + "github.com/hashicorp/terraform-ls/internal/terraform/datadir" +) + +// Watcher is a wrapper around native fsnotify.Watcher +// It provides the ability to detect actual file changes +// (rather than just events that may not be changing any bytes) +type watcher struct { + fw *fsnotify.Watcher + fs filesystem.Filesystem + modMgr ModuleManager + modules []*watchedModule + logger *log.Logger + + watching bool + cancelFunc context.CancelFunc +} + +type WatcherFactory func(filesystem.Filesystem, ModuleManager) (Watcher, error) + +type watchedModule struct { + Path string + Watched []string + Watchable *datadir.WatchablePaths +} + +func NewWatcher(fs filesystem.Filesystem, modMgr ModuleManager) (Watcher, error) { + fw, err := fsnotify.NewWatcher() + if err != nil { + return nil, err + } + + return &watcher{ + fw: fw, + fs: fs, + modMgr: modMgr, + logger: defaultLogger, + modules: make([]*watchedModule, 0), + }, nil +} + +var defaultLogger = log.New(ioutil.Discard, "", 0) + +func (w *watcher) SetLogger(logger *log.Logger) { + w.logger = logger +} + +func (w *watcher) IsModuleWatched(modPath string) bool { + modPath = filepath.Clean(modPath) + + for _, m := range w.modules { + if pathEquals(m.Path, modPath) { + return true + } + } + + return false +} + +func (w *watcher) AddModule(modPath string) error { + modPath = filepath.Clean(modPath) + + w.logger.Printf("adding module for watching: %s", modPath) + + wm := &watchedModule{ + Path: modPath, + Watched: make([]string, 0), + Watchable: datadir.WatchableModulePaths(modPath), + } + w.modules = append(w.modules, wm) + + // We watch individual dirs (instead of individual files). + // This does result in more events but fewer watched paths. + // fsnotify does not support recursive watching yet. + // See https://github.com/fsnotify/fsnotify/issues/18 + + err := w.fw.Add(modPath) + if err != nil { + return err + } + + for _, dirPath := range wm.Watchable.Dirs { + err := w.fw.Add(dirPath) + if err == nil { + wm.Watched = append(wm.Watched, dirPath) + } + } + + return nil +} + +func (w *watcher) run(ctx context.Context) { + for { + select { + case event, ok := <-w.fw.Events: + if !ok { + return + } + w.processEvent(event) + case err, ok := <-w.fw.Errors: + if !ok { + return + } + w.logger.Println("watch error:", err) + } + } +} + +func (w *watcher) processEvent(event fsnotify.Event) { + eventPath := event.Name + + if event.Op&fsnotify.Write == fsnotify.Write { + for _, mod := range w.modules { + if containsPath(mod.Watchable.ModuleManifests, eventPath) { + w.modMgr.EnqueueModuleOp(mod.Path, OpTypeParseModuleManifest) + return + } + if containsPath(mod.Watchable.PluginLockFiles, eventPath) { + w.modMgr.EnqueueModuleOp(mod.Path, OpTypeObtainSchema) + return + } + } + } + + if event.Op&fsnotify.Create == fsnotify.Create { + for _, mod := range w.modules { + if containsPath(mod.Watchable.Dirs, eventPath) { + w.fw.Add(eventPath) + mod.Watched = append(mod.Watched, eventPath) + + filepath.Walk(eventPath, func(path string, info os.FileInfo, err error) error { + if info.IsDir() { + if containsPath(mod.Watchable.Dirs, path) { + w.fw.Add(path) + mod.Watched = append(mod.Watched, path) + } + return nil + } + if containsPath(mod.Watchable.ModuleManifests, path) { + return w.modMgr.EnqueueModuleOp(mod.Path, OpTypeParseModuleManifest) + } + if containsPath(mod.Watchable.PluginLockFiles, path) { + return w.modMgr.EnqueueModuleOp(mod.Path, OpTypeObtainSchema) + } + return nil + }) + + return + } + + if containsPath(mod.Watchable.ModuleManifests, eventPath) { + w.modMgr.EnqueueModuleOp(mod.Path, OpTypeParseModuleManifest) + return + } + + if containsPath(mod.Watchable.PluginLockFiles, eventPath) { + w.modMgr.EnqueueModuleOp(mod.Path, OpTypeObtainSchema) + return + } + } + } + + if event.Op&fsnotify.Remove == fsnotify.Remove { + for modI, mod := range w.modules { + // Whole module being removed + if pathEquals(mod.Path, eventPath) { + for _, wPath := range mod.Watched { + w.fw.Remove(wPath) + } + w.fw.Remove(mod.Path) + w.modules = append(w.modules[:modI], w.modules[modI+1:]...) + return + } + + for i, wp := range mod.Watched { + if pathEquals(wp, eventPath) { + w.fw.Remove(wp) + mod.Watched = append(mod.Watched[:i], mod.Watched[i+1:]...) + return + } + } + } + } +} + +func containsPath(paths []string, path string) bool { + for _, p := range paths { + if pathEquals(p, path) { + return true + } + } + return false +} + +func (w *watcher) Start() error { + if w.watching { + w.logger.Println("watching already in progress") + return nil + } + + ctx, cancelFunc := context.WithCancel(context.Background()) + w.cancelFunc = cancelFunc + w.watching = true + + w.logger.Printf("watching for changes ...") + go w.run(ctx) + + return nil +} + +func (w *watcher) Stop() error { + if !w.watching { + return nil + } + + w.cancelFunc() + + err := w.fw.Close() + if err == nil { + w.watching = false + } + + return err +} diff --git a/internal/watcher/watcher_mock.go b/internal/terraform/module/watcher_mock.go similarity index 52% rename from internal/watcher/watcher_mock.go rename to internal/terraform/module/watcher_mock.go index 8eef0ddf6..bc5b0d9d2 100644 --- a/internal/watcher/watcher_mock.go +++ b/internal/terraform/module/watcher_mock.go @@ -1,34 +1,32 @@ -package watcher +package module import ( "log" + + "github.com/hashicorp/terraform-ls/internal/filesystem" ) func MockWatcher() WatcherFactory { - return func() (Watcher, error) { + return func(filesystem.Filesystem, ModuleManager) (Watcher, error) { return &mockWatcher{}, nil } } type mockWatcher struct{} -func (w *mockWatcher) AddChangeHook(h ChangeHook) { -} - -func (w *mockWatcher) AddPaths(paths []string) error { +func (w *mockWatcher) Start() error { return nil } - -func (w *mockWatcher) AddPath(path string) error { +func (w *mockWatcher) Stop() error { return nil } -func (w *mockWatcher) Start() error { - return nil -} +func (w *mockWatcher) SetLogger(*log.Logger) {} -func (w *mockWatcher) Stop() error { +func (w *mockWatcher) AddModule(string) error { return nil } -func (w *mockWatcher) SetLogger(*log.Logger) {} +func (w *mockWatcher) IsModuleWatched(string) bool { + return false +} diff --git a/internal/terraform/module/watcher_test.go b/internal/terraform/module/watcher_test.go new file mode 100644 index 000000000..10e461ad2 --- /dev/null +++ b/internal/terraform/module/watcher_test.go @@ -0,0 +1,117 @@ +package module + +import ( + "context" + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/hashicorp/terraform-json" + "github.com/hashicorp/terraform-ls/internal/filesystem" + "github.com/hashicorp/terraform-ls/internal/terraform/exec" + "github.com/stretchr/testify/mock" +) + +func TestWatcher_initFromScratch(t *testing.T) { + fs := filesystem.NewFilesystem() + + modPath := filepath.Join(t.TempDir(), "module") + err := os.Mkdir(modPath, 0755) + if err != nil { + t.Fatal(err) + } + + psMock := &tfjson.ProviderSchemas{ + FormatVersion: "0.1", + Schemas: map[string]*tfjson.ProviderSchema{ + "custom": {}, + }, + } + mmm := NewModuleManagerMock(&ModuleManagerMockInput{ + Logger: testLogger(), + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + modPath: { + { + Method: "ProviderSchemas", + Arguments: []interface{}{ + mock.AnythingOfType("*context.cancelCtx"), + }, + ReturnArguments: []interface{}{ + psMock, + nil, + }, + }, + }, + }, + }, + }) + ctx := context.Background() + modMgr := mmm(ctx, fs) + + w, err := NewWatcher(fs, modMgr) + if err != nil { + t.Fatal(err) + } + w.SetLogger(testLogger()) + + mod, err := modMgr.AddModule(modPath) + if err != nil { + t.Fatal(err) + } + + b := []byte(` +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 3.0" + } + } +} + +provider "aws" { + region = "us-east-1" +} + +resource "aws_vpc" "example" { + cidr_block = "10.0.0.0/16" +} +`) + err = ioutil.WriteFile(filepath.Join(modPath, "main.tf"), b, 0755) + if err != nil { + t.Fatal(err) + } + + err = w.AddModule(modPath) + if err != nil { + t.Fatal(err) + } + + err = w.Start() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + w.Stop() + }) + + err = ioutil.WriteFile(filepath.Join(modPath, ".terraform.lock.hcl"), b, 0755) + if err != nil { + t.Fatal(err) + } + + // Give watcher some time to react + time.Sleep(100 * time.Millisecond) + + ps, err := mod.ProviderSchema() + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(psMock, ps); diff != "" { + t.Fatalf("schema mismatch: %s", diff) + } +} diff --git a/internal/watcher/tracked_file.go b/internal/watcher/tracked_file.go deleted file mode 100644 index 861bf6a1a..000000000 --- a/internal/watcher/tracked_file.go +++ /dev/null @@ -1,54 +0,0 @@ -package watcher - -import ( - "crypto/sha256" - "io" - "os" - "path/filepath" -) - -func trackedFileFromPath(path string) (TrackedFile, error) { - path, err := filepath.EvalSymlinks(path) - if err != nil { - return nil, err - } - - b, err := fileSha256Sum(path) - if err != nil { - return nil, err - } - - return &trackedFile{ - path: path, - sha256sum: string(b), - }, nil -} - -type trackedFile struct { - path string - sha256sum string -} - -func (tf *trackedFile) Path() string { - return tf.path -} - -func (tf *trackedFile) Sha256Sum() string { - return tf.sha256sum -} - -func fileSha256Sum(path string) ([]byte, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer f.Close() - - h := sha256.New() - _, err = io.Copy(h, f) - if err != nil { - return nil, err - } - - return h.Sum(nil), nil -} diff --git a/internal/watcher/types.go b/internal/watcher/types.go deleted file mode 100644 index 0555bb23d..000000000 --- a/internal/watcher/types.go +++ /dev/null @@ -1,22 +0,0 @@ -package watcher - -import ( - "context" - "log" -) - -type TrackedFile interface { - Path() string - Sha256Sum() string -} - -type Watcher interface { - Start() error - Stop() error - SetLogger(logger *log.Logger) - AddPath(path string) error - AddPaths(paths []string) error - AddChangeHook(f ChangeHook) -} - -type ChangeHook func(ctx context.Context, file TrackedFile) error diff --git a/internal/watcher/watcher.go b/internal/watcher/watcher.go deleted file mode 100644 index 252859543..000000000 --- a/internal/watcher/watcher.go +++ /dev/null @@ -1,137 +0,0 @@ -package watcher - -import ( - "context" - "io/ioutil" - "log" - - "github.com/fsnotify/fsnotify" -) - -// Watcher is a wrapper around native fsnotify.Watcher -// It provides the ability to detect actual file changes -// (rather than just events that may not be changing any bytes) -type watcher struct { - fw *fsnotify.Watcher - trackedFiles map[string]TrackedFile - changeHooks []ChangeHook - logger *log.Logger - - watching bool - cancelFunc context.CancelFunc -} - -type WatcherFactory func() (Watcher, error) - -func NewWatcher() (Watcher, error) { - fw, err := fsnotify.NewWatcher() - if err != nil { - return nil, err - } - return &watcher{ - fw: fw, - logger: defaultLogger, - trackedFiles: make(map[string]TrackedFile, 0), - }, nil -} - -var defaultLogger = log.New(ioutil.Discard, "", 0) - -func (w *watcher) SetLogger(logger *log.Logger) { - w.logger = logger -} - -func (w *watcher) AddPaths(paths []string) error { - for _, p := range paths { - err := w.AddPath(p) - if err != nil { - return err - } - } - return nil -} - -func (w *watcher) AddPath(path string) error { - w.logger.Printf("adding %s for watching", path) - - tf, err := trackedFileFromPath(path) - if err != nil { - return err - } - w.trackedFiles[path] = tf - - return w.fw.Add(path) -} - -func (w *watcher) AddChangeHook(h ChangeHook) { - w.changeHooks = append(w.changeHooks, h) -} - -func (w *watcher) run(ctx context.Context) { - for { - select { - case event, ok := <-w.fw.Events: - if !ok { - return - } - - if event.Op&fsnotify.Write == fsnotify.Write { - w.logger.Printf("detected write into %s", event.Name) - oldTf := w.trackedFiles[event.Name] - newTf, err := trackedFileFromPath(event.Name) - if err != nil { - w.logger.Println("failed to track file, ignoring", err) - continue - } - w.trackedFiles[event.Name] = newTf - - if oldTf.Sha256Sum() != newTf.Sha256Sum() { - for _, h := range w.changeHooks { - err := h(ctx, newTf) - if err != nil { - w.logger.Println("change hook error:", err) - } - } - } - } - case err, ok := <-w.fw.Errors: - if !ok { - return - } - w.logger.Println("watch error:", err) - } - } -} - -// StartWatching starts to watch for changes that were added -// via AddPath(s) until Stop() is called -func (w *watcher) Start() error { - if w.watching { - w.logger.Println("watching already in progress") - return nil - } - - ctx, cancelFunc := context.WithCancel(context.Background()) - w.cancelFunc = cancelFunc - w.watching = true - - w.logger.Printf("watching for changes ...") - go w.run(ctx) - - return nil -} - -func (w *watcher) Stop() error { - if !w.watching { - return nil - } - - w.cancelFunc() - - err := w.fw.Close() - if err == nil { - w.watching = false - } - - return err -}