From 0d2f8060042cb68747fb6b59387e93ff9897b12a Mon Sep 17 00:00:00 2001 From: Radek Simko Date: Wed, 20 Jan 2021 15:15:33 +0000 Subject: [PATCH] watcher: Detect 'terraform init' from scratch This is a bigger change than perhaps expected but I was not able to come up with any decent way of splitting it up. As mentioned in one in-line comment, there is currently no reliable and efficient way of watching a directory *recursively* in Go. There are some alternative solutions, but they either involve cgo or don't actually use any "event API" of the underlying OS but repeatedly do filewalk. Previously we watched individual manifest and lock file of a module and we can't watch for *creation* of these files in a directory which doesn't exist yet. Therefore it was decided to instead watch the relevant parent directories and add relevant nested directories on creation and/or walk the hierarchy of the newly created dir. This creates even more places from where parts of module can be (re)loaded (walker, watcher, handlers), which would be hard to accommodate in the old design. Therefore a mechanism for queueing all module operations was created. This allows for a naive deduplication of operations (we don't run the same operation on the same module if it's already queued) and also allows us to report progress more easily in the future. Finally this design also allows any caller to decide whether run an operation synchronously or asynchronously. e.g. We can wait for a module to be parsed in textDocument/didChange handler. --- internal/cmd/completion_command.go | 16 +- internal/cmd/inspect_module_command.go | 47 +- internal/context/context.go | 41 +- internal/filesystem/filesystem.go | 28 +- internal/filesystem/filesystem_metadata.go | 23 + internal/filesystem/filesystem_test.go | 108 +++ internal/filesystem/types.go | 2 + .../langserver/diagnostics/validate_diags.go | 45 + internal/langserver/handlers/command/init.go | 31 +- .../langserver/handlers/command/modules.go | 30 +- .../langserver/handlers/command/validate.go | 31 +- internal/langserver/handlers/complete.go | 8 +- internal/langserver/handlers/complete_test.go | 15 +- internal/langserver/handlers/did_change.go | 9 +- .../langserver/handlers/did_change_test.go | 9 +- internal/langserver/handlers/did_open.go | 46 +- .../handlers/execute_command_init_test.go | 27 +- .../handlers/execute_command_modules_test.go | 29 +- .../handlers/execute_command_test.go | 9 +- .../handlers/execute_command_validate_test.go | 9 +- internal/langserver/handlers/formatting.go | 43 +- .../langserver/handlers/formatting_test.go | 17 +- internal/langserver/handlers/handlers_test.go | 34 +- internal/langserver/handlers/hover.go | 6 +- internal/langserver/handlers/hover_test.go | 12 +- internal/langserver/handlers/initialize.go | 35 +- .../langserver/handlers/initialize_test.go | 32 +- .../langserver/handlers/semantic_tokens.go | 6 +- .../handlers/semantic_tokens_test.go | 17 +- internal/langserver/handlers/service.go | 97 +- ...vice_mock_test.go => session_mock_test.go} | 25 +- internal/langserver/handlers/shutdown_test.go | 13 +- internal/langserver/handlers/symbols.go | 6 +- internal/langserver/handlers/symbols_test.go | 9 +- internal/langserver/handlers/tick_reporter.go | 40 - internal/terraform/datadir/datadir.go | 71 ++ .../{module => datadir}/module_manifest.go | 40 +- .../terraform/datadir/module_manifest_test.go | 116 +++ .../module_manifest_unix_test.go | 2 +- .../module_manifest_windows_test.go | 2 +- internal/terraform/datadir/paths.go | 30 + .../terraform/datadir/plugin_lock_file.go | 19 + internal/terraform/exec/exec.go | 2 + internal/terraform/exec/exec_mock.go | 40 +- internal/terraform/exec/exec_opts.go | 23 + internal/terraform/module/file.go | 43 - internal/terraform/module/module.go | 845 ++++-------------- internal/terraform/module/module_loader.go | 163 ++++ internal/terraform/module/module_manager.go | 398 +++------ .../terraform/module/module_manager_mock.go | 98 +- .../module/module_manager_mock_test.go | 74 -- .../terraform/module/module_manager_test.go | 292 ++---- .../terraform/module/module_manifest_test.go | 27 - internal/terraform/module/module_mock.go | 30 - internal/terraform/module/module_ops.go | 214 +++++ internal/terraform/module/module_ops_queue.go | 99 ++ .../terraform/module/module_ops_queue_test.go | 91 ++ internal/terraform/module/path.go | 2 + internal/terraform/module/plugin_lock_file.go | 26 - .../terraform/module/terraform_executor.go | 42 + internal/terraform/module/types.go | 101 +-- internal/terraform/module/walker.go | 79 +- internal/terraform/module/walker_mock.go | 6 +- internal/terraform/module/watcher.go | 234 +++++ .../module}/watcher_mock.go | 24 +- internal/terraform/module/watcher_test.go | 117 +++ internal/watcher/tracked_file.go | 54 -- internal/watcher/types.go | 22 - internal/watcher/watcher.go | 137 --- 69 files changed, 2405 insertions(+), 2113 deletions(-) create mode 100644 internal/langserver/diagnostics/validate_diags.go rename internal/langserver/handlers/{service_mock_test.go => session_mock_test.go} (75%) delete mode 100644 internal/langserver/handlers/tick_reporter.go create mode 100644 internal/terraform/datadir/datadir.go rename internal/terraform/{module => datadir}/module_manifest.go (76%) create mode 100644 internal/terraform/datadir/module_manifest_test.go rename internal/terraform/{module => datadir}/module_manifest_unix_test.go (99%) rename internal/terraform/{module => datadir}/module_manifest_windows_test.go (99%) create mode 100644 internal/terraform/datadir/paths.go create mode 100644 internal/terraform/datadir/plugin_lock_file.go create mode 100644 internal/terraform/exec/exec_opts.go delete mode 100644 internal/terraform/module/file.go create mode 100644 internal/terraform/module/module_loader.go delete mode 100644 internal/terraform/module/module_manager_mock_test.go delete mode 100644 internal/terraform/module/module_manifest_test.go delete mode 100644 internal/terraform/module/module_mock.go create mode 100644 internal/terraform/module/module_ops.go create mode 100644 internal/terraform/module/module_ops_queue.go create mode 100644 internal/terraform/module/module_ops_queue_test.go delete mode 100644 internal/terraform/module/plugin_lock_file.go create mode 100644 internal/terraform/module/terraform_executor.go create mode 100644 internal/terraform/module/watcher.go rename internal/{watcher => terraform/module}/watcher_mock.go (52%) create mode 100644 internal/terraform/module/watcher_test.go delete mode 100644 internal/watcher/tracked_file.go delete mode 100644 internal/watcher/types.go delete mode 100644 internal/watcher/watcher.go diff --git a/internal/cmd/completion_command.go b/internal/cmd/completion_command.go index 0d428df6b..4c3717e85 100644 --- a/internal/cmd/completion_command.go +++ b/internal/cmd/completion_command.go @@ -102,21 +102,27 @@ func (c *CompletionCommand) Run(args []string) int { return 1 } - module, err := module.NewModule(context.Background(), fs, fh.Dir()) + ctx := context.Background() + modMgr := module.NewSyncModuleManager(ctx, fs) + + mod, err := modMgr.AddModule(fh.Dir()) if err != nil { - c.Ui.Error(fmt.Sprintf("failed to load module: %s", err.Error())) + c.Ui.Error(err.Error()) return 1 } - schema, err := module.MergedSchema() + + schema, err := modMgr.SchemaForModule(fh.Dir()) if err != nil { c.Ui.Error(fmt.Sprintf("failed to find schema: %s", err.Error())) return 1 } - d, err := module.DecoderWithSchema(schema) + + d, err := module.DecoderForModule(mod) if err != nil { - c.Ui.Error(fmt.Sprintf("failed to find parser: %s", err.Error())) + c.Ui.Error(fmt.Sprintf("failed to find decoder: %s", err.Error())) return 1 } + d.SetSchema(schema) pos := fPos.Position() diff --git a/internal/cmd/inspect_module_command.go b/internal/cmd/inspect_module_command.go index dad97446c..22b7ddfbe 100644 --- a/internal/cmd/inspect_module_command.go +++ b/internal/cmd/inspect_module_command.go @@ -16,6 +16,7 @@ import ( ictx "github.com/hashicorp/terraform-ls/internal/context" "github.com/hashicorp/terraform-ls/internal/filesystem" "github.com/hashicorp/terraform-ls/internal/logging" + "github.com/hashicorp/terraform-ls/internal/terraform/datadir" "github.com/hashicorp/terraform-ls/internal/terraform/module" "github.com/mitchellh/cli" ) @@ -84,46 +85,50 @@ func (c *InspectModuleCommand) inspect(rootPath string) error { fs := filesystem.NewFilesystem() - modMgr := module.NewModuleManager(fs) + ctx := context.Background() + modMgr := module.NewSyncModuleManager(ctx, fs) modMgr.SetLogger(c.logger) - walker := module.NewWalker() + + walker := module.SyncWalker(fs, modMgr) walker.SetLogger(c.logger) ctx, cancel := ictx.WithSignalCancel(context.Background(), c.logger, syscall.SIGINT, syscall.SIGTERM) defer cancel() - err = walker.StartWalking(ctx, rootPath, func(ctx context.Context, dir string) error { - mod, err := modMgr.AddAndStartLoadingModule(ctx, dir) - if err != nil { - return err - } - <-mod.LoadingDone() - - return nil - }) + err = walker.StartWalking(ctx, rootPath) if err != nil { return err } - <-walker.Done() - modules := modMgr.ListModules() c.Ui.Output(fmt.Sprintf("%d modules found in total at %s", len(modules), rootPath)) for _, mod := range modules { errs := &multierror.Error{} - err := mod.LoadError() + _, err = mod.TerraformVersion() if err != nil { - var ok bool - errs, ok = err.(*multierror.Error) - if !ok { - return err - } + multierror.Append(errs, err) + } + + _, err := mod.ProviderSchema() + if err != nil { + multierror.Append(errs, err) + } + + _, err = mod.ModuleManifest() + if err != nil { + multierror.Append(errs, err) } + + _, err = mod.ParsedFiles() + if err != nil { + multierror.Append(errs, err) + } + errs.ErrorFormat = formatErrors - modules := formatModuleRecords(mod.Modules()) + modules := formatModuleRecords(mod.ModuleCalls()) subModules := fmt.Sprintf("%d modules", len(modules)) if len(modules) > 0 { subModules += "\n" @@ -153,7 +158,7 @@ func formatErrors(errors []error) string { return strings.TrimSpace(out) } -func formatModuleRecords(mds []module.ModuleRecord) []string { +func formatModuleRecords(mds []datadir.ModuleRecord) []string { out := make([]string, 0) for _, m := range mds { if m.IsRoot() { diff --git a/internal/context/context.go b/internal/context/context.go index 753eb4fa5..8f8a65d8b 100644 --- a/internal/context/context.go +++ b/internal/context/context.go @@ -9,7 +9,6 @@ import ( lsp "github.com/hashicorp/terraform-ls/internal/protocol" "github.com/hashicorp/terraform-ls/internal/settings" "github.com/hashicorp/terraform-ls/internal/terraform/module" - "github.com/hashicorp/terraform-ls/internal/watcher" ) type contextKey struct { @@ -29,10 +28,8 @@ var ( ctxTfExecTimeout = &contextKey{"terraform execution timeout"} ctxWatcher = &contextKey{"watcher"} ctxModuleMngr = &contextKey{"module manager"} - ctxTfFormatterFinder = &contextKey{"terraform formatter finder"} - ctxModuleCaFi = &contextKey{"module candidate finder"} + ctxModuleFinder = &contextKey{"module finder"} ctxModuleWalker = &contextKey{"module walker"} - ctxModuleLoader = &contextKey{"module loader"} ctxRootDir = &contextKey{"root directory"} ctxCommandPrefix = &contextKey{"command prefix"} ctxDiags = &contextKey{"diagnostics"} @@ -103,12 +100,12 @@ func TerraformExecTimeout(ctx context.Context) (time.Duration, bool) { return path, ok } -func WithWatcher(ctx context.Context, w watcher.Watcher) context.Context { +func WithWatcher(ctx context.Context, w module.Watcher) context.Context { return context.WithValue(ctx, ctxWatcher, w) } -func Watcher(ctx context.Context) (watcher.Watcher, error) { - w, ok := ctx.Value(ctxWatcher).(watcher.Watcher) +func Watcher(ctx context.Context) (module.Watcher, error) { + w, ok := ctx.Value(ctxWatcher).(module.Watcher) if !ok { return nil, missingContextErr(ctxWatcher) } @@ -127,18 +124,6 @@ func ModuleManager(ctx context.Context) (module.ModuleManager, error) { return wm, nil } -func WithTerraformFormatterFinder(ctx context.Context, tef module.TerraformFormatterFinder) context.Context { - return context.WithValue(ctx, ctxTfFormatterFinder, tef) -} - -func TerraformFormatterFinder(ctx context.Context) (module.TerraformFormatterFinder, error) { - pf, ok := ctx.Value(ctxTfFormatterFinder).(module.TerraformFormatterFinder) - if !ok { - return nil, missingContextErr(ctxTfFormatterFinder) - } - return pf, nil -} - func WithTerraformExecPath(ctx context.Context, path string) context.Context { return context.WithValue(ctx, ctxTfExecPath, path) } @@ -149,13 +134,13 @@ func TerraformExecPath(ctx context.Context) (string, bool) { } func WithModuleFinder(ctx context.Context, mf module.ModuleFinder) context.Context { - return context.WithValue(ctx, ctxModuleCaFi, mf) + return context.WithValue(ctx, ctxModuleFinder, mf) } func ModuleFinder(ctx context.Context) (module.ModuleFinder, error) { - cf, ok := ctx.Value(ctxModuleCaFi).(module.ModuleFinder) + cf, ok := ctx.Value(ctxModuleFinder).(module.ModuleFinder) if !ok { - return nil, missingContextErr(ctxModuleCaFi) + return nil, missingContextErr(ctxModuleFinder) } return cf, nil } @@ -216,18 +201,6 @@ func ModuleWalker(ctx context.Context) (*module.Walker, error) { return w, nil } -func WithModuleLoader(ctx context.Context, ml module.ModuleLoader) context.Context { - return context.WithValue(ctx, ctxModuleLoader, ml) -} - -func ModuleLoader(ctx context.Context) (module.ModuleLoader, error) { - w, ok := ctx.Value(ctxModuleLoader).(module.ModuleLoader) - if !ok { - return nil, missingContextErr(ctxModuleLoader) - } - return w, nil -} - func WithDiagnostics(ctx context.Context, diags *diagnostics.Notifier) context.Context { return context.WithValue(ctx, ctxDiags, diags) } diff --git a/internal/filesystem/filesystem.go b/internal/filesystem/filesystem.go index b564beee2..0d89922a1 100644 --- a/internal/filesystem/filesystem.go +++ b/internal/filesystem/filesystem.go @@ -2,6 +2,7 @@ package filesystem import ( "bytes" + "fmt" "io/ioutil" "log" "os" @@ -36,6 +37,18 @@ func (fs *fsystem) SetLogger(logger *log.Logger) { } func (fs *fsystem) CreateDocument(dh DocumentHandler, text []byte) error { + _, err := fs.memFs.Stat(dh.Dir()) + if err != nil { + if os.IsNotExist(err) { + err := fs.memFs.MkdirAll(dh.Dir(), 0755) + if err != nil { + return fmt.Errorf("failed to create parent dir: %w", err) + } + } else { + return err + } + } + f, err := fs.memFs.Create(dh.FullPath()) if err != nil { return err @@ -196,11 +209,11 @@ func (fs *fsystem) ReadFile(name string) ([]byte, error) { func (fs *fsystem) ReadDir(name string) ([]os.FileInfo, error) { memList, err := afero.ReadDir(fs.memFs, name) if err != nil && !os.IsNotExist(err) { - return nil, err + return nil, fmt.Errorf("memory FS: %w", err) } osList, err := afero.ReadDir(fs.osFs, name) - if err != nil { - return nil, err + if err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("OS FS: %w", err) } list := memList @@ -231,3 +244,12 @@ func (fs *fsystem) Open(name string) (File, error) { return f, err } + +func (fs *fsystem) Stat(name string) (os.FileInfo, error) { + fi, err := fs.memFs.Stat(name) + if err != nil && os.IsNotExist(err) { + return fs.osFs.Stat(name) + } + + return fi, err +} diff --git a/internal/filesystem/filesystem_metadata.go b/internal/filesystem/filesystem_metadata.go index b4166b7d0..ccad86eec 100644 --- a/internal/filesystem/filesystem_metadata.go +++ b/internal/filesystem/filesystem_metadata.go @@ -1,7 +1,10 @@ package filesystem import ( + "path/filepath" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/terraform-ls/internal/uri" ) func (fs *fsystem) markDocumentAsOpen(dh DocumentHandler) error { @@ -16,6 +19,26 @@ func (fs *fsystem) markDocumentAsOpen(dh DocumentHandler) error { return nil } +func (fs *fsystem) HasOpenFiles(dirPath string) (bool, error) { + files, err := fs.ReadDir(dirPath) + if err != nil { + return false, err + } + + fs.docMetaMu.RLock() + defer fs.docMetaMu.RUnlock() + + for _, fi := range files { + u := uri.FromPath(filepath.Join(dirPath, fi.Name())) + dm, ok := fs.docMeta[u] + if ok && dm.IsOpen() { + return true, nil + } + } + + return false, nil +} + func (fs *fsystem) createDocumentMetadata(dh DocumentHandler, text []byte) error { if fs.documentMetadataExists(dh) { return &MetadataAlreadyExistsErr{dh} diff --git a/internal/filesystem/filesystem_test.go b/internal/filesystem/filesystem_test.go index bdbe9ddb9..eae40a5fb 100644 --- a/internal/filesystem/filesystem_test.go +++ b/internal/filesystem/filesystem_test.go @@ -294,6 +294,29 @@ func TestFilesystem_ReadDir(t *testing.T) { } } +func TestFilesystem_ReadDir_memFsOnly(t *testing.T) { + fs := NewFilesystem() + + tmpDir := t.TempDir() + + fh := testHandlerFromPath(filepath.Join(tmpDir, "memfile")) + err := fs.CreateDocument(fh, []byte("test")) + if err != nil { + t.Fatal(err) + } + + fis, err := fs.ReadDir(tmpDir) + if err != nil { + t.Fatal(err) + } + + expectedFis := []string{"memfile"} + names := namesFromFileInfos(fis) + if diff := cmp.Diff(expectedFis, names); diff != "" { + t.Fatalf("file list mismatch: %s", diff) + } +} + func namesFromFileInfos(fis []os.FileInfo) []string { names := make([]string, len(fis), len(fis)) for i, fi := range fis { @@ -433,6 +456,86 @@ func TestFilesystem_Create_memOnly(t *testing.T) { } } +func TestFilesystem_CreateDocument_missingParentDir(t *testing.T) { + fs := NewFilesystem() + + tmpDir := t.TempDir() + testPath := filepath.Join(tmpDir, "foo", "bar", "test.tf") + fh := testHandlerFromPath(testPath) + + err := fs.CreateDocument(fh, []byte("test")) + if err != nil { + t.Fatal(err) + } + + fooPath := filepath.Join(tmpDir, "foo") + fi, err := fs.memFs.Stat(fooPath) + if err != nil { + t.Fatal(err) + } + if !fi.IsDir() { + t.Fatalf("expected %q to be a dir", fooPath) + } + + barPath := filepath.Join(tmpDir, "foo", "bar") + fi, err = fs.memFs.Stat(barPath) + if err != nil { + t.Fatal(err) + } + if !fi.IsDir() { + t.Fatalf("expected %q to be a dir", barPath) + } +} + +func TestFilesystem_HasOpenFiles(t *testing.T) { + fs := NewFilesystem() + + tmpDir := t.TempDir() + + notOpenHandler := filepath.Join(tmpDir, "not-open.tf") + noFh := testHandlerFromPath(notOpenHandler) + err := fs.CreateDocument(noFh, []byte("test1")) + if err != nil { + t.Fatal(err) + } + + of, err := fs.HasOpenFiles(tmpDir) + if err != nil { + t.Fatal(err) + } + if of { + t.Fatalf("expected no open files for %s", tmpDir) + } + + openHandler := filepath.Join(tmpDir, "open.tf") + ofh := testHandlerFromPath(openHandler) + err = fs.CreateAndOpenDocument(ofh, []byte("test2")) + if err != nil { + t.Fatal(err) + } + + of, err = fs.HasOpenFiles(tmpDir) + if err != nil { + t.Fatal(err) + } + if !of { + t.Fatalf("expected open files for %s", tmpDir) + } + + err = fs.CloseAndRemoveDocument(ofh) + if err != nil { + t.Fatal(err) + } + + of, err = fs.HasOpenFiles(tmpDir) + if err != nil { + t.Fatal(err) + } + if of { + t.Fatalf("expected no open files for %s", tmpDir) + } +} + func TempDir(t *testing.T) string { tmpDir := filepath.Join(os.TempDir(), "terraform-ls", t.Name()) @@ -478,6 +581,11 @@ func (fh *testHandler) Dir() string { func (fh *testHandler) Filename() string { return "" } + +func (fh *testHandler) IsOpen() bool { + return false +} + func (fh *testHandler) Version() int { return 0 } diff --git a/internal/filesystem/types.go b/internal/filesystem/types.go index cc9b9e77f..f8df53cf1 100644 --- a/internal/filesystem/types.go +++ b/internal/filesystem/types.go @@ -40,6 +40,7 @@ type DocumentStorage interface { GetDocument(DocumentHandler) (Document, error) CloseAndRemoveDocument(DocumentHandler) error ChangeDocument(VersionedDocumentHandler, DocumentChanges) error + HasOpenFiles(path string) (bool, error) } type Filesystem interface { @@ -51,6 +52,7 @@ type Filesystem interface { ReadFile(name string) ([]byte, error) ReadDir(name string) ([]os.FileInfo, error) Open(name string) (File, error) + Stat(name string) (os.FileInfo, error) } // File represents an open file in FS diff --git a/internal/langserver/diagnostics/validate_diags.go b/internal/langserver/diagnostics/validate_diags.go new file mode 100644 index 000000000..c7564ed52 --- /dev/null +++ b/internal/langserver/diagnostics/validate_diags.go @@ -0,0 +1,45 @@ +package diagnostics + +import ( + "github.com/hashicorp/hcl/v2" + tfjson "github.com/hashicorp/terraform-json" +) + +// tfjson.Diagnostic is a conversion of an internal diag to terraform core, +// tfdiags, which is effectively based on hcl.Diagnostic. +// This process is really just converting it back to hcl.Diagnotic +// since it is the defacto diagnostic type for our codebase currently +// https://github.com/hashicorp/terraform/blob/ae025248cc0712bf53c675dc2fe77af4276dd5cc/command/validate.go#L138 +func HCLDiagsFromJSON(jsonDiags []tfjson.Diagnostic) map[string]hcl.Diagnostics { + diagsMap := make(map[string]hcl.Diagnostics) + + for _, d := range jsonDiags { + // the diagnostic must be tied to a file to exist in the map + if d.Range == nil || d.Range.Filename == "" { + continue + } + + diags := diagsMap[d.Range.Filename] + + var severity hcl.DiagnosticSeverity + if d.Severity == "error" { + severity = hcl.DiagError + } else if d.Severity == "warning" { + severity = hcl.DiagWarning + } + + diags = append(diags, &hcl.Diagnostic{ + Severity: severity, + Summary: d.Summary, + Detail: d.Detail, + Subject: &hcl.Range{ + Filename: d.Range.Filename, + Start: hcl.Pos(d.Range.Start), + End: hcl.Pos(d.Range.End), + }, + }) + diagsMap[d.Range.Filename] = diags + } + + return diagsMap +} diff --git a/internal/langserver/handlers/command/init.go b/internal/langserver/handlers/command/init.go index 84e7861b3..6714f50e1 100644 --- a/internal/langserver/handlers/command/init.go +++ b/internal/langserver/handlers/command/init.go @@ -10,6 +10,7 @@ import ( "github.com/hashicorp/terraform-ls/internal/langserver/progress" ilsp "github.com/hashicorp/terraform-ls/internal/lsp" lsp "github.com/hashicorp/terraform-ls/internal/protocol" + "github.com/hashicorp/terraform-ls/internal/terraform/module" ) func TerraformInitHandler(ctx context.Context, args cmd.CommandArgs) (interface{}, error) { @@ -20,12 +21,24 @@ func TerraformInitHandler(ctx context.Context, args cmd.CommandArgs) (interface{ dh := ilsp.FileHandlerFromDirURI(lsp.DocumentURI(dirUri)) - cf, err := lsctx.ModuleFinder(ctx) + modMgr, err := lsctx.ModuleManager(ctx) if err != nil { return nil, err } - mod, err := cf.ModuleByPath(dh.Dir()) + mod, err := modMgr.ModuleByPath(dh.Dir()) + if err != nil { + if module.IsModuleNotFound(err) { + mod, err = modMgr.AddModule(dh.Dir()) + if err != nil { + return nil, err + } + } else { + return nil, err + } + } + + tfExec, err := module.TerraformExecutorForModule(ctx, mod) if err != nil { return nil, err } @@ -36,22 +49,10 @@ func TerraformInitHandler(ctx context.Context, args cmd.CommandArgs) (interface{ }() progress.Report(ctx, "Running terraform init ...") - err = mod.ExecuteTerraformInit(ctx) - if err != nil { - return nil, err - } - - progress.Report(ctx, "Detecting paths to watch ...") - paths := mod.PathsToWatch() - - w, err := lsctx.Watcher(ctx) + err = tfExec.Init(ctx) if err != nil { return nil, err } - err = w.AddPaths(paths) - if err != nil { - return nil, fmt.Errorf("failed to add watch for dir (%s): %+v", dh.Dir(), err) - } return nil, nil } diff --git a/internal/langserver/handlers/command/modules.go b/internal/langserver/handlers/command/modules.go index a8ccbb62b..56cb085ae 100644 --- a/internal/langserver/handlers/command/modules.go +++ b/internal/langserver/handlers/command/modules.go @@ -10,6 +10,7 @@ import ( "github.com/hashicorp/terraform-ls/internal/langserver/cmd" ilsp "github.com/hashicorp/terraform-ls/internal/lsp" lsp "github.com/hashicorp/terraform-ls/internal/protocol" + "github.com/hashicorp/terraform-ls/internal/terraform/module" "github.com/hashicorp/terraform-ls/internal/uri" ) @@ -39,19 +40,36 @@ func ModulesHandler(ctx context.Context, args cmd.CommandArgs) (interface{}, err fh := ilsp.FileHandlerFromDocumentURI(lsp.DocumentURI(fileUri)) - cf, err := lsctx.ModuleFinder(ctx) + modMgr, err := lsctx.ModuleManager(ctx) if err != nil { return nil, err } + doneLoading := !walker.IsWalking() - candidates := cf.ModuleCandidatesByPath(fh.Dir()) + + var sources []module.SchemaSource + sources, err = modMgr.SchemaSourcesForModule(fh.Dir()) + if err != nil { + if module.IsModuleNotFound(err) { + _, err := modMgr.AddModule(fh.Dir()) + if err != nil { + return nil, err + } + sources, err = modMgr.SchemaSourcesForModule(fh.Dir()) + if err != nil { + return nil, err + } + } else { + return nil, err + } + } rootDir, _ := lsctx.RootDirectory(ctx) - modules := make([]moduleInfo, len(candidates)) - for i, candidate := range candidates { + modules := make([]moduleInfo, len(sources)) + for i, source := range sources { modules[i] = moduleInfo{ - URI: uri.FromPath(candidate.Path()), - Name: candidate.HumanReadablePath(rootDir), + URI: uri.FromPath(source.Path()), + Name: source.HumanReadablePath(rootDir), } } sort.SliceStable(modules, func(i, j int) bool { diff --git a/internal/langserver/handlers/command/validate.go b/internal/langserver/handlers/command/validate.go index 5d125c0a3..46efe0001 100644 --- a/internal/langserver/handlers/command/validate.go +++ b/internal/langserver/handlers/command/validate.go @@ -7,9 +7,11 @@ import ( "github.com/creachadair/jrpc2/code" lsctx "github.com/hashicorp/terraform-ls/internal/context" "github.com/hashicorp/terraform-ls/internal/langserver/cmd" + "github.com/hashicorp/terraform-ls/internal/langserver/diagnostics" "github.com/hashicorp/terraform-ls/internal/langserver/progress" ilsp "github.com/hashicorp/terraform-ls/internal/lsp" lsp "github.com/hashicorp/terraform-ls/internal/protocol" + "github.com/hashicorp/terraform-ls/internal/terraform/module" ) func TerraformValidateHandler(ctx context.Context, args cmd.CommandArgs) (interface{}, error) { @@ -20,25 +22,29 @@ func TerraformValidateHandler(ctx context.Context, args cmd.CommandArgs) (interf dh := ilsp.FileHandlerFromDirURI(lsp.DocumentURI(dirUri)) - cf, err := lsctx.ModuleFinder(ctx) + modMgr, err := lsctx.ModuleManager(ctx) if err != nil { return nil, err } - mod, err := cf.ModuleByPath(dh.Dir()) + mod, err := modMgr.ModuleByPath(dh.Dir()) if err != nil { - return nil, err + if module.IsModuleNotFound(err) { + mod, err = modMgr.AddModule(dh.Dir()) + if err != nil { + return nil, err + } + } else { + return nil, err + } } - wasInit, err := mod.WasInitialized() + tfExec, err := module.TerraformExecutorForModule(ctx, mod) if err != nil { - return nil, fmt.Errorf("error checking if %s was initialized: %s", dirUri, err) - } - if !wasInit { - return nil, fmt.Errorf("%s is not an initialized module, terraform validate cannot be called", dirUri) + return nil, err } - diags, err := lsctx.Diagnostics(ctx) + notifier, err := lsctx.Diagnostics(ctx) if err != nil { return nil, err } @@ -48,11 +54,14 @@ func TerraformValidateHandler(ctx context.Context, args cmd.CommandArgs) (interf progress.End(ctx, "Finished") }() progress.Report(ctx, "Running terraform validate ...") - hclDiags, err := mod.ExecuteTerraformValidate(ctx) + jsonDiags, err := tfExec.Validate(ctx) if err != nil { return nil, err } - diags.PublishHCLDiags(ctx, mod.Path(), hclDiags, "terraform validate") + + diags := diagnostics.HCLDiagsFromJSON(jsonDiags) + + notifier.PublishHCLDiags(ctx, mod.Path(), diags, "terraform validate") return nil, nil } diff --git a/internal/langserver/handlers/complete.go b/internal/langserver/handlers/complete.go index 091a88069..aaf6c0f1f 100644 --- a/internal/langserver/handlers/complete.go +++ b/internal/langserver/handlers/complete.go @@ -6,6 +6,7 @@ import ( lsctx "github.com/hashicorp/terraform-ls/internal/context" ilsp "github.com/hashicorp/terraform-ls/internal/lsp" lsp "github.com/hashicorp/terraform-ls/internal/protocol" + "github.com/hashicorp/terraform-ls/internal/terraform/module" ) func (h *logHandler) TextDocumentComplete(ctx context.Context, params lsp.CompletionParams) (lsp.CompletionList, error) { @@ -31,20 +32,21 @@ func (h *logHandler) TextDocumentComplete(ctx context.Context, params lsp.Comple return list, err } - module, err := mf.ModuleByPath(file.Dir()) + mod, err := mf.ModuleByPath(file.Dir()) if err != nil { return list, err } - schema, err := mf.SchemaForPath(file.Dir()) + schema, err := mf.SchemaForModule(file.Dir()) if err != nil { return list, err } - d, err := module.DecoderWithSchema(schema) + d, err := module.DecoderForModule(mod) if err != nil { return list, err } + d.SetSchema(schema) fPos, err := ilsp.FilePositionFromDocumentPosition(params.TextDocumentPositionParams, file) if err != nil { diff --git a/internal/langserver/handlers/complete_test.go b/internal/langserver/handlers/complete_test.go index cd80d3be5..fc9557528 100644 --- a/internal/langserver/handlers/complete_test.go +++ b/internal/langserver/handlers/complete_test.go @@ -10,7 +10,6 @@ import ( "github.com/hashicorp/terraform-ls/internal/langserver" "github.com/hashicorp/terraform-ls/internal/langserver/session" "github.com/hashicorp/terraform-ls/internal/terraform/exec" - "github.com/hashicorp/terraform-ls/internal/terraform/module" "github.com/stretchr/testify/mock" ) @@ -43,9 +42,9 @@ func TestCompletion_withValidData(t *testing.T) { } ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: exec.NewMockExecutor([]*mock.Call{ + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): []*mock.Call{ { Method: "Version", Repeatability: 1, @@ -76,7 +75,7 @@ func TestCompletion_withValidData(t *testing.T) { nil, }, }, - }), + }, }, }})) stop := ls.Start(t) @@ -88,7 +87,7 @@ func TestCompletion_withValidData(t *testing.T) { "capabilities": {}, "rootUri": %q, "processId": 12345 - }`, TempDir(t).URI())}) + }`, tmpDir.URI())}) ls.Notify(t, &langserver.CallRequest{ Method: "initialized", ReqParams: "{}", @@ -102,7 +101,7 @@ func TestCompletion_withValidData(t *testing.T) { "text": "provider \"test\" {\n\n}\n", "uri": "%s/main.tf" } - }`, TempDir(t).URI())}) + }`, tmpDir.URI())}) ls.CallAndExpectResponse(t, &langserver.CallRequest{ Method: "textDocument/completion", @@ -114,7 +113,7 @@ func TestCompletion_withValidData(t *testing.T) { "character": 0, "line": 1 } - }`, TempDir(t).URI())}, `{ + }`, tmpDir.URI())}, `{ "jsonrpc": "2.0", "id": 3, "result": { diff --git a/internal/langserver/handlers/did_change.go b/internal/langserver/handlers/did_change.go index 17b50399f..8e74cfc2f 100644 --- a/internal/langserver/handlers/did_change.go +++ b/internal/langserver/handlers/did_change.go @@ -7,6 +7,7 @@ import ( lsctx "github.com/hashicorp/terraform-ls/internal/context" ilsp "github.com/hashicorp/terraform-ls/internal/lsp" lsp "github.com/hashicorp/terraform-ls/internal/protocol" + "github.com/hashicorp/terraform-ls/internal/terraform/module" ) func TextDocumentDidChange(ctx context.Context, params lsp.DidChangeTextDocumentParams) error { @@ -48,17 +49,17 @@ func TextDocumentDidChange(ctx context.Context, params lsp.DidChangeTextDocument return err } - mf, err := lsctx.ModuleFinder(ctx) + modMgr, err := lsctx.ModuleManager(ctx) if err != nil { return err } - module, err := mf.ModuleByPath(fh.Dir()) + mod, err := modMgr.ModuleByPath(fh.Dir()) if err != nil { return err } - err = module.ParseFiles() + err = modMgr.EnqueueModuleOpWait(mod.Path(), module.OpTypeParseConfiguration) if err != nil { return err } @@ -67,7 +68,7 @@ func TextDocumentDidChange(ctx context.Context, params lsp.DidChangeTextDocument if err != nil { return err } - diags.PublishHCLDiags(ctx, module.Path(), module.ParsedDiagnostics(), "HCL") + diags.PublishHCLDiags(ctx, mod.Path(), mod.Diagnostics(), "HCL") return nil } diff --git a/internal/langserver/handlers/did_change_test.go b/internal/langserver/handlers/did_change_test.go index 62f9df908..d1316095d 100644 --- a/internal/langserver/handlers/did_change_test.go +++ b/internal/langserver/handlers/did_change_test.go @@ -9,7 +9,8 @@ import ( "github.com/hashicorp/terraform-ls/internal/filesystem" "github.com/hashicorp/terraform-ls/internal/langserver" "github.com/hashicorp/terraform-ls/internal/lsp" - "github.com/hashicorp/terraform-ls/internal/terraform/module" + "github.com/hashicorp/terraform-ls/internal/terraform/exec" + "github.com/stretchr/testify/mock" ) func TestLangServer_didChange_sequenceOfPartialChanges(t *testing.T) { @@ -18,9 +19,9 @@ func TestLangServer_didChange_sequenceOfPartialChanges(t *testing.T) { fs := filesystem.NewFilesystem() ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: validTfMockCalls(), + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), }, }, Filesystem: fs, diff --git a/internal/langserver/handlers/did_open.go b/internal/langserver/handlers/did_open.go index b55b4a016..8c89c6142 100644 --- a/internal/langserver/handlers/did_open.go +++ b/internal/langserver/handlers/did_open.go @@ -46,7 +46,7 @@ func (lh *logHandler) TextDocumentDidOpen(ctx context.Context, params lsp.DidOpe mod, err = modMgr.ModuleByPath(f.Dir()) if err != nil { if module.IsModuleNotFound(err) { - mod, err = modMgr.AddAndStartLoadingModule(ctx, f.Dir()) + mod, err = modMgr.AddModule(f.Dir()) if err != nil { return err } @@ -60,23 +60,39 @@ func (lh *logHandler) TextDocumentDidOpen(ctx context.Context, params lsp.DidOpe // We reparse because the file being opened may not match // (originally parsed) content on the disk // TODO: Do this only if we can verify the file differs? - err = mod.ParseFiles() + modMgr.EnqueueModuleOpWait(mod.Path(), module.OpTypeParseConfiguration) + + if mod.TerraformVersionState() == module.OpStateUnknown { + modMgr.EnqueueModuleOp(mod.Path(), module.OpTypeGetTerraformVersion) + } + + watcher, err := lsctx.Watcher(ctx) if err != nil { - return fmt.Errorf("failed to parse files: %w", err) + return err + } + + if !watcher.IsModuleWatched(mod.Path()) { + err := watcher.AddModule(mod.Path()) + if err != nil { + return err + } } diags, err := lsctx.Diagnostics(ctx) if err != nil { return err } - diags.PublishHCLDiags(ctx, mod.Path(), mod.ParsedDiagnostics(), "HCL") + diags.PublishHCLDiags(ctx, mod.Path(), mod.Diagnostics(), "HCL") - candidates := modMgr.ModuleCandidatesByPath(f.Dir()) + sources, err := modMgr.SchemaSourcesForModule(f.Dir()) + if err != nil { + return err + } - if walker.IsWalking() { - // avoid raising false warnings if walker hasn't finished yet + if walker.IsWalking() || mod.ProviderSchemaState() == module.OpStateLoading { + // avoid raising false warnings if operations are still in-flight lh.logger.Printf("walker has not finished walking yet, data may be inaccurate for %s", f.FullPath()) - } else if len(candidates) == 0 { + } else if len(sources) == 0 { // TODO: Only notify once per f.Dir() per session dh := ilsp.FileHandlerFromDirPath(f.Dir()) go func() { @@ -90,12 +106,12 @@ func (lh *logHandler) TextDocumentDidOpen(ctx context.Context, params lsp.DidOpe }() } - if len(candidates) > 1 { - candidateDir := humanReadablePath(rootDir, candidates[0].Path()) + if len(sources) > 1 { + candidateDir := humanReadablePath(rootDir, sources[0].Path()) msg := fmt.Sprintf("Alternative schema source found for %s (%s), picked: %s."+ " You can set an explicit module path in your settings.", - readableDir, candidatePaths(rootDir, candidates[1:]), + readableDir, candidatePaths(rootDir, sources[1:]), candidateDir) return jrpc2.PushNotify(ctx, "window/showMessage", lsp.ShowMessageParams{ Type: lsp.Warning, @@ -106,10 +122,10 @@ func (lh *logHandler) TextDocumentDidOpen(ctx context.Context, params lsp.DidOpe return nil } -func candidatePaths(rootDir string, candidates []module.Module) string { - paths := make([]string, len(candidates)) - for i, mod := range candidates { - paths[i] = humanReadablePath(rootDir, mod.Path()) +func candidatePaths(rootDir string, sources []module.SchemaSource) string { + paths := make([]string, len(sources)) + for i, source := range sources { + paths[i] = humanReadablePath(rootDir, source.Path()) } return strings.Join(paths, ", ") } diff --git a/internal/langserver/handlers/execute_command_init_test.go b/internal/langserver/handlers/execute_command_init_test.go index 0def28f5e..ec4074bb9 100644 --- a/internal/langserver/handlers/execute_command_init_test.go +++ b/internal/langserver/handlers/execute_command_init_test.go @@ -10,7 +10,6 @@ import ( "github.com/hashicorp/terraform-ls/internal/langserver" "github.com/hashicorp/terraform-ls/internal/langserver/cmd" "github.com/hashicorp/terraform-ls/internal/terraform/exec" - "github.com/hashicorp/terraform-ls/internal/terraform/module" "github.com/stretchr/testify/mock" ) @@ -19,9 +18,9 @@ func TestLangServer_workspaceExecuteCommand_init_argumentError(t *testing.T) { testFileURI := fmt.Sprintf("%s/main.tf", tmpDir.URI()) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: validTfMockCalls(), + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), }, }, })) @@ -61,7 +60,7 @@ func TestLangServer_workspaceExecuteCommand_init_basic(t *testing.T) { tmpDir := TempDir(t) testFileURI := fmt.Sprintf("%s/main.tf", tmpDir.URI()) - tfMockCalls := exec.NewMockExecutor([]*mock.Call{ + tfMockCalls := []*mock.Call{ { Method: "Version", Repeatability: 1, @@ -91,12 +90,12 @@ func TestLangServer_workspaceExecuteCommand_init_basic(t *testing.T) { nil, }, }, - }) + } ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: tfMockCalls, + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): tfMockCalls, }, }, })) @@ -141,7 +140,7 @@ func TestLangServer_workspaceExecuteCommand_init_error(t *testing.T) { tmpDir := TempDir(t) testFileURI := fmt.Sprintf("%s/main.tf", tmpDir.URI()) - tfMockCalls := exec.NewMockExecutor([]*mock.Call{ + tfMockCalls := []*mock.Call{ { Method: "Version", Repeatability: 1, @@ -171,12 +170,12 @@ func TestLangServer_workspaceExecuteCommand_init_error(t *testing.T) { errors.New("something bad happened"), }, }, - }) + } ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: tfMockCalls, + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): tfMockCalls, }, }, })) diff --git a/internal/langserver/handlers/execute_command_modules_test.go b/internal/langserver/handlers/execute_command_modules_test.go index 7582ceac5..e700797de 100644 --- a/internal/langserver/handlers/execute_command_modules_test.go +++ b/internal/langserver/handlers/execute_command_modules_test.go @@ -9,7 +9,8 @@ import ( "github.com/hashicorp/terraform-ls/internal/langserver" "github.com/hashicorp/terraform-ls/internal/langserver/cmd" "github.com/hashicorp/terraform-ls/internal/lsp" - "github.com/hashicorp/terraform-ls/internal/terraform/module" + "github.com/hashicorp/terraform-ls/internal/terraform/exec" + "github.com/stretchr/testify/mock" ) func TestLangServer_workspaceExecuteCommand_modules_argumentError(t *testing.T) { @@ -18,9 +19,9 @@ func TestLangServer_workspaceExecuteCommand_modules_argumentError(t *testing.T) InitPluginCache(t, tmpDir.Dir()) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: validTfMockCalls(), + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), }, }, })) @@ -62,9 +63,9 @@ func TestLangServer_workspaceExecuteCommand_modules_basic(t *testing.T) { InitPluginCache(t, tmpDir.Dir()) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: validTfMockCalls(), + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), }, }, })) @@ -128,15 +129,11 @@ func TestLangServer_workspaceExecuteCommand_modules_multiple(t *testing.T) { prod := lsp.FileHandlerFromDirPath(filepath.Join(testData, "main-module-multienv", "env", "prod")) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - dev.Dir(): { - TfExecFactory: validTfMockCalls(), - }, - staging.Dir(): { - TfExecFactory: validTfMockCalls(), - }, - prod.Dir(): { - TfExecFactory: validTfMockCalls(), + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + dev.Dir(): validTfMockCalls(), + staging.Dir(): validTfMockCalls(), + prod.Dir(): validTfMockCalls(), }, }, })) diff --git a/internal/langserver/handlers/execute_command_test.go b/internal/langserver/handlers/execute_command_test.go index 2bb52bf9e..7d7d24be2 100644 --- a/internal/langserver/handlers/execute_command_test.go +++ b/internal/langserver/handlers/execute_command_test.go @@ -6,7 +6,8 @@ import ( "github.com/creachadair/jrpc2/code" "github.com/hashicorp/terraform-ls/internal/langserver" - "github.com/hashicorp/terraform-ls/internal/terraform/module" + "github.com/hashicorp/terraform-ls/internal/terraform/exec" + "github.com/stretchr/testify/mock" ) func TestLangServer_workspaceExecuteCommand_noCommandHandlerError(t *testing.T) { @@ -16,9 +17,9 @@ func TestLangServer_workspaceExecuteCommand_noCommandHandlerError(t *testing.T) InitPluginCache(t, tmpDir.Dir()) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: validTfMockCalls(), + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), }, }, })) diff --git a/internal/langserver/handlers/execute_command_validate_test.go b/internal/langserver/handlers/execute_command_validate_test.go index 95d313fb0..c6cfeaebb 100644 --- a/internal/langserver/handlers/execute_command_validate_test.go +++ b/internal/langserver/handlers/execute_command_validate_test.go @@ -7,7 +7,8 @@ import ( "github.com/creachadair/jrpc2/code" "github.com/hashicorp/terraform-ls/internal/langserver" "github.com/hashicorp/terraform-ls/internal/langserver/cmd" - "github.com/hashicorp/terraform-ls/internal/terraform/module" + "github.com/hashicorp/terraform-ls/internal/terraform/exec" + "github.com/stretchr/testify/mock" ) func TestLangServer_workspaceExecuteCommand_validate_argumentError(t *testing.T) { @@ -15,9 +16,9 @@ func TestLangServer_workspaceExecuteCommand_validate_argumentError(t *testing.T) testFileURI := fmt.Sprintf("%s/main.tf", tmpDir.URI()) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: validTfMockCalls(), + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), }, }, })) diff --git a/internal/langserver/handlers/formatting.go b/internal/langserver/handlers/formatting.go index e52c0303c..40032501d 100644 --- a/internal/langserver/handlers/formatting.go +++ b/internal/langserver/handlers/formatting.go @@ -2,13 +2,11 @@ package handlers import ( "context" - "fmt" lsctx "github.com/hashicorp/terraform-ls/internal/context" "github.com/hashicorp/terraform-ls/internal/hcl" ilsp "github.com/hashicorp/terraform-ls/internal/lsp" lsp "github.com/hashicorp/terraform-ls/internal/protocol" - "github.com/hashicorp/terraform-ls/internal/terraform/exec" "github.com/hashicorp/terraform-ls/internal/terraform/module" ) @@ -20,18 +18,24 @@ func (h *logHandler) TextDocumentFormatting(ctx context.Context, params lsp.Docu return edits, err } - tff, err := lsctx.TerraformFormatterFinder(ctx) + mf, err := lsctx.ModuleFinder(ctx) if err != nil { return edits, err } fh := ilsp.FileHandlerFromDocumentURI(params.TextDocument.URI) - file, err := fs.GetDocument(fh) + + mod, err := mf.ModuleByPath(fh.Dir()) + if err != nil { + return edits, err + } + + tfExec, err := module.TerraformExecutorForModule(ctx, mod) if err != nil { return edits, err } - format, err := findTerraformFormatter(ctx, tff, file.Dir()) + file, err := fs.GetDocument(fh) if err != nil { return edits, err } @@ -41,7 +45,7 @@ func (h *logHandler) TextDocumentFormatting(ctx context.Context, params lsp.Docu return edits, err } - formatted, err := format(ctx, original) + formatted, err := tfExec.Format(ctx, original) if err != nil { return edits, err } @@ -50,30 +54,3 @@ func (h *logHandler) TextDocumentFormatting(ctx context.Context, params lsp.Docu return ilsp.TextEditsFromDocumentChanges(changes), nil } - -func findTerraformFormatter(ctx context.Context, tff module.TerraformFormatterFinder, dir string) (exec.Formatter, error) { - discoveryDone, err := tff.HasTerraformDiscoveryFinished(dir) - if err != nil { - if module.IsModuleNotFound(err) { - return tff.TerraformFormatterForDir(ctx, dir) - } - return nil, err - } else { - if !discoveryDone { - // TODO: block until it's available <-tff.TerraformLoadingDone() - return nil, fmt.Errorf("terraform is still being discovered for %s", dir) - } - available, err := tff.IsTerraformAvailable(dir) - if err != nil { - if module.IsModuleNotFound(err) { - return tff.TerraformFormatterForDir(ctx, dir) - } - } - if !available { - // TODO: block until it's available <-tff.TerraformLoadingDone() - return nil, fmt.Errorf("terraform is not available for %s", dir) - } - } - - return tff.TerraformFormatterForDir(ctx, dir) -} diff --git a/internal/langserver/handlers/formatting_test.go b/internal/langserver/handlers/formatting_test.go index 923597d3d..8c407a027 100644 --- a/internal/langserver/handlers/formatting_test.go +++ b/internal/langserver/handlers/formatting_test.go @@ -10,7 +10,6 @@ import ( "github.com/hashicorp/terraform-ls/internal/langserver" "github.com/hashicorp/terraform-ls/internal/langserver/session" "github.com/hashicorp/terraform-ls/internal/terraform/exec" - "github.com/hashicorp/terraform-ls/internal/terraform/module" "github.com/stretchr/testify/mock" ) @@ -35,9 +34,9 @@ func TestLangServer_formatting_basic(t *testing.T) { tmpDir := TempDir(t) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: exec.NewMockExecutor([]*mock.Call{ + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): { { Method: "Version", Repeatability: 1, @@ -69,7 +68,7 @@ func TestLangServer_formatting_basic(t *testing.T) { nil, }, }, - }), + }, }, }, })) @@ -121,9 +120,9 @@ func TestLangServer_formatting_basic(t *testing.T) { func TestLangServer_formatting_oldVersion(t *testing.T) { tmpDir := TempDir(t) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: exec.NewMockExecutor([]*mock.Call{ + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): { { Method: "Version", Repeatability: 1, @@ -155,7 +154,7 @@ func TestLangServer_formatting_oldVersion(t *testing.T) { errors.New("not implemented"), }, }, - }), + }, }, }, })) diff --git a/internal/langserver/handlers/handlers_test.go b/internal/langserver/handlers/handlers_test.go index 0e981873d..e97ca636d 100644 --- a/internal/langserver/handlers/handlers_test.go +++ b/internal/langserver/handlers/handlers_test.go @@ -13,7 +13,6 @@ import ( "github.com/hashicorp/terraform-ls/internal/langserver" "github.com/hashicorp/terraform-ls/internal/lsp" "github.com/hashicorp/terraform-ls/internal/terraform/exec" - "github.com/hashicorp/terraform-ls/internal/terraform/module" "github.com/stretchr/testify/mock" ) @@ -69,9 +68,12 @@ func TestInitalizeAndShutdown(t *testing.T) { tmpDir := TempDir(t) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): {TfExecFactory: validTfMockCalls()}, - }})) + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), + }, + }, + })) stop := ls.Start(t) defer stop() @@ -95,9 +97,12 @@ func TestInitalizeWithCommandPrefix(t *testing.T) { tmpDir := TempDir(t) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): {TfExecFactory: validTfMockCalls()}, - }})) + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), + }, + }, + })) stop := ls.Start(t) defer stop() @@ -117,9 +122,12 @@ func TestEOF(t *testing.T) { tmpDir := TempDir(t) ms := newMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - TempDir(t).Dir(): {TfExecFactory: validTfMockCalls()}, - }}) + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), + }, + }, + }) ls := langserver.NewLangServerMock(t, ms.new) stop := ls.Start(t) defer stop() @@ -146,8 +154,8 @@ func TestEOF(t *testing.T) { } } -func validTfMockCalls() exec.ExecutorFactory { - return exec.NewMockExecutor([]*mock.Call{ +func validTfMockCalls() []*mock.Call { + return []*mock.Call{ { Method: "Version", Repeatability: 1, @@ -178,7 +186,7 @@ func validTfMockCalls() exec.ExecutorFactory { nil, }, }, - }) + } } // TempDir creates a temporary directory containing the test name, as well any diff --git a/internal/langserver/handlers/hover.go b/internal/langserver/handlers/hover.go index cf588a215..b654e3348 100644 --- a/internal/langserver/handlers/hover.go +++ b/internal/langserver/handlers/hover.go @@ -6,6 +6,7 @@ import ( lsctx "github.com/hashicorp/terraform-ls/internal/context" ilsp "github.com/hashicorp/terraform-ls/internal/lsp" lsp "github.com/hashicorp/terraform-ls/internal/protocol" + "github.com/hashicorp/terraform-ls/internal/terraform/module" ) func (h *logHandler) TextDocumentHover(ctx context.Context, params lsp.TextDocumentPositionParams) (*lsp.Hover, error) { @@ -34,15 +35,16 @@ func (h *logHandler) TextDocumentHover(ctx context.Context, params lsp.TextDocum return nil, err } - schema, err := mf.SchemaForPath(file.Dir()) + schema, err := mf.SchemaForModule(file.Dir()) if err != nil { return nil, err } - d, err := mod.DecoderWithSchema(schema) + d, err := module.DecoderForModule(mod) if err != nil { return nil, err } + d.SetSchema(schema) fPos, err := ilsp.FilePositionFromDocumentPosition(params, file) if err != nil { diff --git a/internal/langserver/handlers/hover_test.go b/internal/langserver/handlers/hover_test.go index 74812864e..de57ee3b6 100644 --- a/internal/langserver/handlers/hover_test.go +++ b/internal/langserver/handlers/hover_test.go @@ -10,7 +10,6 @@ import ( "github.com/hashicorp/terraform-ls/internal/langserver" "github.com/hashicorp/terraform-ls/internal/langserver/session" "github.com/hashicorp/terraform-ls/internal/terraform/exec" - "github.com/hashicorp/terraform-ls/internal/terraform/module" "github.com/stretchr/testify/mock" ) @@ -43,9 +42,9 @@ func TestHover_withValidData(t *testing.T) { } ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: exec.NewMockExecutor([]*mock.Call{ + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): { { Method: "Version", Repeatability: 1, @@ -76,9 +75,10 @@ func TestHover_withValidData(t *testing.T) { nil, }, }, - }), + }, }, - }})) + }, + })) stop := ls.Start(t) defer stop() diff --git a/internal/langserver/handlers/initialize.go b/internal/langserver/handlers/initialize.go index e4e5281d8..d21709f2f 100644 --- a/internal/langserver/handlers/initialize.go +++ b/internal/langserver/handlers/initialize.go @@ -57,16 +57,6 @@ func (lh *logHandler) Initialize(ctx context.Context, params lsp.InitializeParam return serverCaps, err } - modMgr, err := lsctx.ModuleManager(ctx) - if err != nil { - return serverCaps, err - } - - addAndLoadModule, err := lsctx.ModuleLoader(ctx) - if err != nil { - return serverCaps, err - } - w, err := lsctx.Watcher(ctx) if err != nil { return serverCaps, err @@ -128,14 +118,8 @@ func (lh *logHandler) Initialize(ctx context.Context, params lsp.InitializeParam }) continue } - mod, err := addAndLoadModule(modPath) - if err != nil { - return serverCaps, err - } - paths := mod.PathsToWatch() - lh.logger.Printf("Adding %d module paths for watching (%s)", len(paths), modPath) - err = w.AddPaths(paths) + err = w.AddModule(modPath) if err != nil { return serverCaps, err } @@ -164,22 +148,7 @@ func (lh *logHandler) Initialize(ctx context.Context, params lsp.InitializeParam // Walker runs asynchronously so we're intentionally *not* // passing the request context here bCtx := context.Background() - err = walker.StartWalking(bCtx, fh.Dir(), func(ctx context.Context, dir string) error { - lh.logger.Printf("Adding module: %s", dir) - mod, err := modMgr.AddAndStartLoadingModule(ctx, dir) - if err != nil { - return err - } - - paths := mod.PathsToWatch() - lh.logger.Printf("Adding %d paths of module for watching (%s)", len(paths), dir) - err = w.AddPaths(paths) - if err != nil { - return err - } - - return nil - }) + err = walker.StartWalking(bCtx, fh.Dir()) return serverCaps, err } diff --git a/internal/langserver/handlers/initialize_test.go b/internal/langserver/handlers/initialize_test.go index edefbc543..3ba93ceb6 100644 --- a/internal/langserver/handlers/initialize_test.go +++ b/internal/langserver/handlers/initialize_test.go @@ -8,15 +8,18 @@ import ( "github.com/hashicorp/go-version" "github.com/hashicorp/terraform-ls/internal/langserver" "github.com/hashicorp/terraform-ls/internal/terraform/exec" - "github.com/hashicorp/terraform-ls/internal/terraform/module" "github.com/stretchr/testify/mock" ) func TestInitialize_twice(t *testing.T) { + tmpDir := TempDir(t) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - TempDir(t).Dir(): {TfExecFactory: validTfMockCalls()}, - }})) + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), + }, + }, + })) stop := ls.Start(t) defer stop() @@ -39,9 +42,9 @@ func TestInitialize_twice(t *testing.T) { func TestInitialize_withIncompatibleTerraformVersion(t *testing.T) { tmpDir := TempDir(t) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: exec.NewMockExecutor([]*mock.Call{ + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): { { Method: "Version", Repeatability: 1, @@ -53,9 +56,10 @@ func TestInitialize_withIncompatibleTerraformVersion(t *testing.T) { nil, }, }, - }), + }, }, - }})) + }, + })) stop := ls.Start(t) defer stop() @@ -69,10 +73,14 @@ func TestInitialize_withIncompatibleTerraformVersion(t *testing.T) { } func TestInitialize_withInvalidRootURI(t *testing.T) { + tmpDir := TempDir(t) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - TempDir(t).Dir(): {TfExecFactory: validTfMockCalls()}, - }})) + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), + }, + }, + })) stop := ls.Start(t) defer stop() diff --git a/internal/langserver/handlers/semantic_tokens.go b/internal/langserver/handlers/semantic_tokens.go index 62cefab46..9ad706d0b 100644 --- a/internal/langserver/handlers/semantic_tokens.go +++ b/internal/langserver/handlers/semantic_tokens.go @@ -8,6 +8,7 @@ import ( lsctx "github.com/hashicorp/terraform-ls/internal/context" ilsp "github.com/hashicorp/terraform-ls/internal/lsp" lsp "github.com/hashicorp/terraform-ls/internal/protocol" + "github.com/hashicorp/terraform-ls/internal/terraform/module" ) func (lh *logHandler) TextDocumentSemanticTokensFull(ctx context.Context, params lsp.SemanticTokensParams) (lsp.SemanticTokens, error) { @@ -50,15 +51,16 @@ func (lh *logHandler) TextDocumentSemanticTokensFull(ctx context.Context, params return tks, fmt.Errorf("finding compatible decoder failed: %w", err) } - schema, err := mf.SchemaForPath(doc.Dir()) + schema, err := mf.SchemaForModule(doc.Dir()) if err != nil { return tks, err } - d, err := mod.DecoderWithSchema(schema) + d, err := module.DecoderForModule(mod) if err != nil { return tks, err } + d.SetSchema(schema) tokens, err := d.SemanticTokensInFile(doc.Filename()) if err != nil { diff --git a/internal/langserver/handlers/semantic_tokens_test.go b/internal/langserver/handlers/semantic_tokens_test.go index ae971785d..ff0633a76 100644 --- a/internal/langserver/handlers/semantic_tokens_test.go +++ b/internal/langserver/handlers/semantic_tokens_test.go @@ -9,7 +9,6 @@ import ( tfjson "github.com/hashicorp/terraform-json" "github.com/hashicorp/terraform-ls/internal/langserver" "github.com/hashicorp/terraform-ls/internal/terraform/exec" - "github.com/hashicorp/terraform-ls/internal/terraform/module" "github.com/stretchr/testify/mock" ) @@ -24,9 +23,9 @@ func TestSemanticTokensFull(t *testing.T) { } ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: exec.NewMockExecutor([]*mock.Call{ + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): { { Method: "Version", Repeatability: 1, @@ -57,7 +56,7 @@ func TestSemanticTokensFull(t *testing.T) { nil, }, }, - }), + }, }, }})) stop := ls.Start(t) @@ -131,9 +130,9 @@ func TestSemanticTokensFull_clientSupportsDelta(t *testing.T) { } ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: exec.NewMockExecutor([]*mock.Call{ + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): { { Method: "Version", Repeatability: 1, @@ -164,7 +163,7 @@ func TestSemanticTokensFull_clientSupportsDelta(t *testing.T) { nil, }, }, - }), + }, }, }})) stop := ls.Start(t) diff --git a/internal/langserver/handlers/service.go b/internal/langserver/handlers/service.go index 90da02b04..c19150c05 100644 --- a/internal/langserver/handlers/service.go +++ b/internal/langserver/handlers/service.go @@ -6,7 +6,6 @@ import ( "fmt" "io/ioutil" "log" - "time" "github.com/creachadair/jrpc2" "github.com/creachadair/jrpc2/code" @@ -17,8 +16,9 @@ import ( "github.com/hashicorp/terraform-ls/internal/langserver/session" lsp "github.com/hashicorp/terraform-ls/internal/protocol" "github.com/hashicorp/terraform-ls/internal/settings" + "github.com/hashicorp/terraform-ls/internal/terraform/discovery" + "github.com/hashicorp/terraform-ls/internal/terraform/exec" "github.com/hashicorp/terraform-ls/internal/terraform/module" - "github.com/hashicorp/terraform-ls/internal/watcher" ) type service struct { @@ -30,18 +30,21 @@ type service struct { stopSession context.CancelFunc fs filesystem.Filesystem - watcher watcher.Watcher + watcher module.Watcher walker *module.Walker modMgr module.ModuleManager newModuleManager module.ModuleManagerFactory - newWatcher watcher.WatcherFactory + newWatcher module.WatcherFactory newWalker module.WalkerFactory + tfDiscoFunc discovery.DiscoveryFunc + tfExecFactory exec.ExecutorFactory } var discardLogs = log.New(ioutil.Discard, "", 0) func NewSession(srvCtx context.Context) session.Session { fs := filesystem.NewFilesystem() + d := &discovery.Discovery{} sessCtx, stopSession := context.WithCancel(srvCtx) return &service{ @@ -51,8 +54,10 @@ func NewSession(srvCtx context.Context) session.Session { sessCtx: sessCtx, stopSession: stopSession, newModuleManager: module.NewModuleManager, - newWatcher: watcher.NewWatcher, + newWatcher: module.NewWatcher, newWalker: module.NewWalker, + tfDiscoFunc: d.LookPath, + tfExecFactory: exec.NewExecutor, } } @@ -77,76 +82,42 @@ func (svc *service) Assigner() (jrpc2.Assigner, error) { lh := LogHandler(svc.logger) cc := &lsp.ClientCapabilities{} - svc.modMgr = svc.newModuleManager(svc.fs) - svc.modMgr.SetLogger(svc.logger) - - svc.logger.Printf("Worker pool size set to %d", svc.modMgr.WorkerPoolSize()) - - tr := newTickReporter(5 * time.Second) - tr.AddReporter(func() { - queueSize := svc.modMgr.WorkerQueueSize() - if queueSize < 1 { - return - } - svc.logger.Printf("Root modules waiting to be loaded: %d", queueSize) - }) - tr.StartReporting(svc.sessCtx) - - svc.walker = svc.newWalker() - // The following is set via CLI flags, hence available in the server context + execOpts := &exec.ExecutorOpts{} if path, ok := lsctx.TerraformExecPath(svc.srvCtx); ok { - svc.modMgr.SetTerraformExecPath(path) + execOpts.ExecPath = path + } else { + tfExecPath, err := svc.tfDiscoFunc() + if err == nil { + execOpts.ExecPath = tfExecPath + } } if path, ok := lsctx.TerraformExecLogPath(svc.srvCtx); ok { - svc.modMgr.SetTerraformExecLogPath(path) + execOpts.ExecLogPath = path } if timeout, ok := lsctx.TerraformExecTimeout(svc.srvCtx); ok { - svc.modMgr.SetTerraformExecTimeout(timeout) + execOpts.Timeout = timeout } - ww, err := svc.newWatcher() + svc.sessCtx = exec.WithExecutorOpts(svc.sessCtx, execOpts) + svc.sessCtx = exec.WithExecutorFactory(svc.sessCtx, svc.tfExecFactory) + + svc.modMgr = svc.newModuleManager(svc.sessCtx, svc.fs) + svc.modMgr.SetLogger(svc.logger) + + svc.walker = svc.newWalker(svc.fs, svc.modMgr) + + ww, err := svc.newWatcher(svc.fs, svc.modMgr) if err != nil { return nil, err } svc.watcher = ww svc.watcher.SetLogger(svc.logger) - svc.watcher.AddChangeHook(func(ctx context.Context, file watcher.TrackedFile) error { - mod, err := svc.modMgr.ModuleByPath(file.Path()) - if err != nil { - return err - } - if mod.IsKnownPluginLockFile(file.Path()) { - svc.logger.Printf("detected plugin cache change, updating schema ...") - err := mod.UpdateProviderSchemaCache(ctx, file) - if err != nil { - svc.logger.Printf(err.Error()) - } - } - - return nil - }) - svc.watcher.AddChangeHook(func(_ context.Context, file watcher.TrackedFile) error { - mod, err := svc.modMgr.ModuleByPath(file.Path()) - if err != nil { - return err - } - if mod.IsKnownModuleManifestFile(file.Path()) { - svc.logger.Printf("detected module manifest change, updating ...") - err := mod.UpdateModuleManifest(file) - if err != nil { - svc.logger.Printf(err.Error()) - } - } - - return nil - }) err = svc.watcher.Start() if err != nil { return nil, err } - modLoader := module.NewModuleLoader(svc.sessCtx, svc.modMgr) diags := diagnostics.NewNotifier(svc.sessCtx, svc.logger) rootDir := "" @@ -166,7 +137,6 @@ func (svc *service) Assigner() (jrpc2.Assigner, error) { ctx = lsctx.WithRootDirectory(ctx, &rootDir) ctx = lsctx.WithCommandPrefix(ctx, &commandPrefix) ctx = lsctx.WithModuleManager(ctx, svc.modMgr) - ctx = lsctx.WithModuleLoader(ctx, modLoader) ctx = lsctx.WithExperimentalFeatures(ctx, &expFeatures) version, ok := lsctx.LanguageServerVersion(svc.srvCtx) @@ -191,7 +161,7 @@ func (svc *service) Assigner() (jrpc2.Assigner, error) { } ctx = lsctx.WithDiagnostics(ctx, diags) ctx = lsctx.WithDocumentStorage(ctx, svc.fs) - ctx = lsctx.WithModuleFinder(ctx, svc.modMgr) + ctx = lsctx.WithModuleManager(ctx, svc.modMgr) return handle(ctx, req, TextDocumentDidChange) }, "textDocument/didOpen": func(ctx context.Context, req *jrpc2.Request) (interface{}, error) { @@ -259,7 +229,9 @@ func (svc *service) Assigner() (jrpc2.Assigner, error) { } ctx = lsctx.WithDocumentStorage(ctx, svc.fs) - ctx = lsctx.WithTerraformFormatterFinder(ctx, svc.modMgr) + ctx = lsctx.WithModuleFinder(ctx, svc.modMgr) + ctx = exec.WithExecutorOpts(ctx, execOpts) + ctx = exec.WithExecutorFactory(ctx, svc.tfExecFactory) return handle(ctx, req, lh.TextDocumentFormatting) }, @@ -284,6 +256,7 @@ func (svc *service) Assigner() (jrpc2.Assigner, error) { ctx = lsctx.WithDiagnostics(ctx, diags) ctx = lsctx.WithExperimentalFeatures(ctx, &expFeatures) ctx = lsctx.WithModuleFinder(ctx, svc.modMgr) + ctx = exec.WithExecutorOpts(ctx, execOpts) return handle(ctx, req, lh.TextDocumentDidSave) }, @@ -294,11 +267,13 @@ func (svc *service) Assigner() (jrpc2.Assigner, error) { } ctx = lsctx.WithCommandPrefix(ctx, &commandPrefix) - ctx = lsctx.WithModuleFinder(ctx, svc.modMgr) + ctx = lsctx.WithModuleManager(ctx, svc.modMgr) ctx = lsctx.WithModuleWalker(ctx, svc.walker) ctx = lsctx.WithWatcher(ctx, ww) ctx = lsctx.WithRootDirectory(ctx, &rootDir) ctx = lsctx.WithDiagnostics(ctx, diags) + ctx = exec.WithExecutorOpts(ctx, execOpts) + ctx = exec.WithExecutorFactory(ctx, svc.tfExecFactory) return handle(ctx, req, lh.WorkspaceExecuteCommand) }, diff --git a/internal/langserver/handlers/service_mock_test.go b/internal/langserver/handlers/session_mock_test.go similarity index 75% rename from internal/langserver/handlers/service_mock_test.go rename to internal/langserver/handlers/session_mock_test.go index cb570a6a9..345aaf9a6 100644 --- a/internal/langserver/handlers/service_mock_test.go +++ b/internal/langserver/handlers/session_mock_test.go @@ -9,15 +9,14 @@ import ( "github.com/hashicorp/terraform-ls/internal/filesystem" "github.com/hashicorp/terraform-ls/internal/langserver/session" + "github.com/hashicorp/terraform-ls/internal/terraform/discovery" "github.com/hashicorp/terraform-ls/internal/terraform/exec" "github.com/hashicorp/terraform-ls/internal/terraform/module" - "github.com/hashicorp/terraform-ls/internal/watcher" ) type MockSessionInput struct { - Modules map[string]*module.ModuleMock - Filesystem filesystem.Filesystem - TfExecutorFactory exec.ExecutorFactory + Filesystem filesystem.Filesystem + TerraformCalls *exec.TerraformMockCalls } type mockSession struct { @@ -34,8 +33,7 @@ func (ms *mockSession) new(srvCtx context.Context) session.Session { var input *module.ModuleManagerMockInput if ms.mockInput != nil { input = &module.ModuleManagerMockInput{ - Modules: ms.mockInput.Modules, - TfExecutorFactory: ms.mockInput.TfExecutorFactory, + Logger: testLogger(), } } @@ -46,6 +44,15 @@ func (ms *mockSession) new(srvCtx context.Context) session.Session { fs = filesystem.NewFilesystem() } + var tfCalls *exec.TerraformMockCalls + if ms.mockInput != nil && ms.mockInput.TerraformCalls != nil { + tfCalls = ms.mockInput.TerraformCalls + } + + d := &discovery.MockDiscovery{ + Path: "tf-mock", + } + svc := &service{ logger: testLogger(), srvCtx: srvCtx, @@ -53,8 +60,10 @@ func (ms *mockSession) new(srvCtx context.Context) session.Session { stopSession: ms.stop, fs: fs, newModuleManager: module.NewModuleManagerMock(input), - newWatcher: watcher.MockWatcher(), - newWalker: module.MockWalker, + newWatcher: module.MockWatcher(), + newWalker: module.SyncWalker, + tfDiscoFunc: d.LookPath, + tfExecFactory: exec.NewMockExecutor(tfCalls), } return svc diff --git a/internal/langserver/handlers/shutdown_test.go b/internal/langserver/handlers/shutdown_test.go index c99541d0c..959480905 100644 --- a/internal/langserver/handlers/shutdown_test.go +++ b/internal/langserver/handlers/shutdown_test.go @@ -6,14 +6,19 @@ import ( "github.com/creachadair/jrpc2/code" "github.com/hashicorp/terraform-ls/internal/langserver" - "github.com/hashicorp/terraform-ls/internal/terraform/module" + "github.com/hashicorp/terraform-ls/internal/terraform/exec" + "github.com/stretchr/testify/mock" ) func TestShutdown_twice(t *testing.T) { + tmpDir := TempDir(t) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - TempDir(t).Dir(): {TfExecFactory: validTfMockCalls()}, - }})) + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), + }, + }, + })) stop := ls.Start(t) defer stop() diff --git a/internal/langserver/handlers/symbols.go b/internal/langserver/handlers/symbols.go index fa8c8daa0..1bf99a3f2 100644 --- a/internal/langserver/handlers/symbols.go +++ b/internal/langserver/handlers/symbols.go @@ -2,11 +2,11 @@ package handlers import ( "context" - "fmt" lsctx "github.com/hashicorp/terraform-ls/internal/context" ilsp "github.com/hashicorp/terraform-ls/internal/lsp" lsp "github.com/hashicorp/terraform-ls/internal/protocol" + "github.com/hashicorp/terraform-ls/internal/terraform/module" ) func (h *logHandler) TextDocumentSymbol(ctx context.Context, params lsp.DocumentSymbolParams) ([]lsp.SymbolInformation, error) { @@ -29,10 +29,10 @@ func (h *logHandler) TextDocumentSymbol(ctx context.Context, params lsp.Document mod, err := mf.ModuleByPath(file.Dir()) if err != nil { - return symbols, fmt.Errorf("finding compatible decoder failed: %w", err) + return symbols, err } - d, err := mod.Decoder() + d, err := module.DecoderForModule(mod) if err != nil { return symbols, err } diff --git a/internal/langserver/handlers/symbols_test.go b/internal/langserver/handlers/symbols_test.go index 621b659c0..cb89023c1 100644 --- a/internal/langserver/handlers/symbols_test.go +++ b/internal/langserver/handlers/symbols_test.go @@ -5,7 +5,8 @@ import ( "testing" "github.com/hashicorp/terraform-ls/internal/langserver" - "github.com/hashicorp/terraform-ls/internal/terraform/module" + "github.com/hashicorp/terraform-ls/internal/terraform/exec" + "github.com/stretchr/testify/mock" ) func TestLangServer_symbols_basic(t *testing.T) { @@ -13,9 +14,9 @@ func TestLangServer_symbols_basic(t *testing.T) { InitPluginCache(t, tmpDir.Dir()) ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{ - Modules: map[string]*module.ModuleMock{ - tmpDir.Dir(): { - TfExecFactory: validTfMockCalls(), + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + tmpDir.Dir(): validTfMockCalls(), }, }, })) diff --git a/internal/langserver/handlers/tick_reporter.go b/internal/langserver/handlers/tick_reporter.go deleted file mode 100644 index 55a8e8e26..000000000 --- a/internal/langserver/handlers/tick_reporter.go +++ /dev/null @@ -1,40 +0,0 @@ -package handlers - -import ( - "context" - "time" -) - -func newTickReporter(d time.Duration) *tickReporter { - return &tickReporter{ - t: time.NewTicker(d), - rfs: make([]reportFunc, 0), - } -} - -type reportFunc func() - -type tickReporter struct { - t *time.Ticker - rfs []reportFunc -} - -func (tr *tickReporter) AddReporter(f reportFunc) { - tr.rfs = append(tr.rfs, f) -} - -func (tr *tickReporter) StartReporting(ctx context.Context) { - go func(ctx context.Context, tr *tickReporter) { - for { - select { - case <-ctx.Done(): - tr.t.Stop() - return - case <-tr.t.C: - for _, rf := range tr.rfs { - rf() - } - } - } - }(ctx, tr) -} diff --git a/internal/terraform/datadir/datadir.go b/internal/terraform/datadir/datadir.go new file mode 100644 index 000000000..8948e3b8b --- /dev/null +++ b/internal/terraform/datadir/datadir.go @@ -0,0 +1,71 @@ +package datadir + +import ( + "path/filepath" + "strings" + + "github.com/hashicorp/terraform-ls/internal/filesystem" +) + +type DataDir struct { + ModuleManifestPath string + PluginLockFilePath string +} + +type WatchablePaths struct { + Dirs []string + ModuleManifests []string + PluginLockFiles []string +} + +func WatchableModulePaths(modPath string) *WatchablePaths { + wp := &WatchablePaths{ + Dirs: watchableModuleDirs(modPath), + ModuleManifests: make([]string, 0), + PluginLockFiles: make([]string, 0), + } + + manifestPath := filepath.Join(append([]string{modPath}, manifestPathElements...)...) + wp.ModuleManifests = append(wp.ModuleManifests, manifestPath) + + for _, pathElems := range pluginLockFilePathElements { + filePath := filepath.Join(append([]string{modPath}, pathElems...)...) + wp.PluginLockFiles = append(wp.PluginLockFiles, filePath) + } + + return wp +} + +// ModulePath strips known lock file paths to get the path +// to the (closest) module these files belong to +func ModulePath(filePath string) (string, bool) { + manifestSuffix := filepath.Join(manifestPathElements...) + if strings.HasSuffix(filePath, manifestSuffix) { + return strings.TrimSuffix(filePath, manifestSuffix), true + } + + for _, pathElems := range pluginLockFilePathElements { + suffix := filepath.Join(pathElems...) + if strings.HasSuffix(filePath, suffix) { + return strings.TrimSuffix(filePath, suffix), true + } + } + + return "", false +} + +func WalkDataDirOfModule(fs filesystem.Filesystem, modPath string) *DataDir { + dir := &DataDir{} + + path, ok := ModuleManifestFilePath(fs, modPath) + if ok { + dir.ModuleManifestPath = path + } + + path, ok = PluginLockFilePath(fs, modPath) + if ok { + dir.PluginLockFilePath = path + } + + return dir +} diff --git a/internal/terraform/module/module_manifest.go b/internal/terraform/datadir/module_manifest.go similarity index 76% rename from internal/terraform/module/module_manifest.go rename to internal/terraform/datadir/module_manifest.go index 079e80a4a..bc3bb707c 100644 --- a/internal/terraform/module/module_manifest.go +++ b/internal/terraform/datadir/module_manifest.go @@ -1,4 +1,4 @@ -package module +package datadir import ( "encoding/json" @@ -8,14 +8,19 @@ import ( "strings" version "github.com/hashicorp/go-version" + "github.com/hashicorp/terraform-ls/internal/filesystem" ) -func moduleManifestFilePath(dir string) string { - return filepath.Join( - dir, - ".terraform", - "modules", - "modules.json") +func ModuleManifestFilePath(fs filesystem.Filesystem, modulePath string) (string, bool) { + manifestPath := filepath.Join( + append([]string{modulePath}, + manifestPathElements...)...) + + fi, err := fs.Stat(manifestPath) + if err == nil && fi.Mode().IsRegular() { + return manifestPath, true + } + return "", false } // The following structs were copied from terraform's @@ -90,15 +95,16 @@ func (r *ModuleRecord) IsExternal() bool { return false } -// moduleManifest is an internal struct used only to assist in our JSON -// serialization of manifest snapshots. It should not be used for any other -// purpose. -type moduleManifest struct { +type ModuleManifest struct { rootDir string Records []ModuleRecord `json:"Modules"` } -func ParseModuleManifestFromFile(path string) (*moduleManifest, error) { +func (mm *ModuleManifest) RootDir() string { + return mm.rootDir +} + +func ParseModuleManifestFromFile(path string) (*ModuleManifest, error) { b, err := ioutil.ReadFile(path) if err != nil { return nil, err @@ -108,13 +114,17 @@ func ParseModuleManifestFromFile(path string) (*moduleManifest, error) { if err != nil { return nil, err } - mm.rootDir = trimLockFilePath(path) + rootDir, ok := ModulePath(path) + if !ok { + return nil, fmt.Errorf("failed to detect module path: %s", path) + } + mm.rootDir = filepath.Clean(rootDir) return mm, nil } -func parseModuleManifest(b []byte) (*moduleManifest, error) { - mm := moduleManifest{} +func parseModuleManifest(b []byte) (*ModuleManifest, error) { + mm := ModuleManifest{} err := json.Unmarshal(b, &mm) if err != nil { return nil, err diff --git a/internal/terraform/datadir/module_manifest_test.go b/internal/terraform/datadir/module_manifest_test.go new file mode 100644 index 000000000..fb75daedd --- /dev/null +++ b/internal/terraform/datadir/module_manifest_test.go @@ -0,0 +1,116 @@ +package datadir + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/hashicorp/go-version" +) + +func TestParseModuleManifestFromFile(t *testing.T) { + modPath := t.TempDir() + manifestDir := filepath.Join(modPath, ".terraform", "modules") + err := os.MkdirAll(manifestDir, 0755) + if err != nil { + t.Fatal(err) + } + + expectedManifest := &ModuleManifest{ + rootDir: modPath, + Records: []ModuleRecord{ + { + Key: "web_server_sg1", + SourceAddr: "terraform-aws-modules/security-group/aws//modules/http-80", + VersionStr: "3.10.0", + Version: version.Must(version.NewVersion("3.10.0")), + Dir: filepath.Join(".terraform", "modules", "web_server_sg", "terraform-aws-security-group-3.10.0", "modules", "http-80"), + }, + { + Key: "web_server_sg2", + SourceAddr: "terraform-aws-modules/security-group/aws//modules/http-80", + VersionStr: "3.10.0", + Version: version.Must(version.NewVersion("3.10.0")), + Dir: filepath.Join(".terraform", "modules", "web_server_sg", "terraform-aws-security-group-3.10.0", "modules", "http-80"), + }, + { + Dir: ".", + }, + { + Key: "local", + SourceAddr: "./nested/path", + Dir: filepath.Join("nested", "path"), + }, + }, + } + + path := filepath.Join(manifestDir, "modules.json") + err = ioutil.WriteFile(path, []byte(testManifestContent), 0755) + if err != nil { + t.Fatal(err) + } + mm, err := ParseModuleManifestFromFile(path) + if err != nil { + t.Fatal(err) + } + + opts := cmp.AllowUnexported(ModuleManifest{}) + if diff := cmp.Diff(expectedManifest, mm, opts); diff != "" { + t.Fatalf("manifest mismatch: %s", diff) + } +} + +const testManifestContent = `{ + "Modules": [ + { + "Key": "web_server_sg1", + "Source": "terraform-aws-modules/security-group/aws//modules/http-80", + "Version": "3.10.0", + "Dir": ".terraform/modules/web_server_sg/terraform-aws-security-group-3.10.0/modules/http-80" + }, + { + "Key": "web_server_sg2", + "Source": "terraform-aws-modules/security-group/aws//modules/http-80", + "Version": "3.10.0", + "Dir": ".terraform/modules/web_server_sg/terraform-aws-security-group-3.10.0/modules/something/../http-80" + }, + { + "Key": "", + "Source": "", + "Dir": "." + }, + { + "Key": "local", + "Source": "./nested/path", + "Dir": "nested/path" + } + ] +}` + +const moduleManifestRecord_external = `{ + "Key": "web_server_sg", + "Source": "terraform-aws-modules/security-group/aws//modules/http-80", + "Version": "3.10.0", + "Dir": ".terraform/modules/web_server_sg/terraform-aws-security-group-3.10.0/modules/http-80" +}` + +const moduleManifestRecord_externalDirtyPath = `{ + "Key": "web_server_sg", + "Source": "terraform-aws-modules/security-group/aws//modules/http-80", + "Version": "3.10.0", + "Dir": ".terraform/modules/web_server_sg/terraform-aws-security-group-3.10.0/modules/something/../http-80" +}` + +const moduleManifestRecord_local = `{ + "Key": "local", + "Source": "./nested/path", + "Dir": "nested/path" +}` + +const moduleManifestRecord_root = `{ + "Key": "", + "Source": "", + "Dir": "." +}` diff --git a/internal/terraform/module/module_manifest_unix_test.go b/internal/terraform/datadir/module_manifest_unix_test.go similarity index 99% rename from internal/terraform/module/module_manifest_unix_test.go rename to internal/terraform/datadir/module_manifest_unix_test.go index c024b7198..a4b4e25f1 100644 --- a/internal/terraform/module/module_manifest_unix_test.go +++ b/internal/terraform/datadir/module_manifest_unix_test.go @@ -1,6 +1,6 @@ // +build !windows -package module +package datadir import ( "encoding/json" diff --git a/internal/terraform/module/module_manifest_windows_test.go b/internal/terraform/datadir/module_manifest_windows_test.go similarity index 99% rename from internal/terraform/module/module_manifest_windows_test.go rename to internal/terraform/datadir/module_manifest_windows_test.go index 77a8071f6..b246712b6 100644 --- a/internal/terraform/module/module_manifest_windows_test.go +++ b/internal/terraform/datadir/module_manifest_windows_test.go @@ -1,4 +1,4 @@ -package module +package datadir import ( "encoding/json" diff --git a/internal/terraform/datadir/paths.go b/internal/terraform/datadir/paths.go new file mode 100644 index 000000000..152ef6b7e --- /dev/null +++ b/internal/terraform/datadir/paths.go @@ -0,0 +1,30 @@ +package datadir + +import ( + "path/filepath" + "runtime" +) + +const DataDirName = ".terraform" + +var pluginLockFilePathElements = [][]string{ + // Terraform >= 0.14 + {".terraform.lock.hcl"}, + // Terraform >= v0.13 + {DataDirName, "plugins", "selections.json"}, + // Terraform >= v0.12 + {DataDirName, "plugins", runtime.GOOS + "_" + runtime.GOARCH, "lock.json"}, +} + +var manifestPathElements = []string{ + DataDirName, "modules", "modules.json", +} + +func watchableModuleDirs(modPath string) []string { + return []string{ + filepath.Join(modPath, DataDirName), + filepath.Join(modPath, DataDirName, "modules"), + filepath.Join(modPath, DataDirName, "plugins"), + filepath.Join(modPath, DataDirName, "plugins", runtime.GOOS+"_"+runtime.GOARCH), + } +} diff --git a/internal/terraform/datadir/plugin_lock_file.go b/internal/terraform/datadir/plugin_lock_file.go new file mode 100644 index 000000000..3e3155480 --- /dev/null +++ b/internal/terraform/datadir/plugin_lock_file.go @@ -0,0 +1,19 @@ +package datadir + +import ( + "path/filepath" + + "github.com/hashicorp/terraform-ls/internal/filesystem" +) + +func PluginLockFilePath(fs filesystem.Filesystem, modPath string) (string, bool) { + for _, pathElems := range pluginLockFilePathElements { + fullPath := filepath.Join(append([]string{modPath}, pathElems...)...) + fi, err := fs.Stat(fullPath) + if err == nil && fi.Mode().IsRegular() { + return fullPath, true + } + } + + return "", false +} diff --git a/internal/terraform/exec/exec.go b/internal/terraform/exec/exec.go index e0b3a8b1f..792a3cfba 100644 --- a/internal/terraform/exec/exec.go +++ b/internal/terraform/exec/exec.go @@ -16,6 +16,8 @@ import ( var defaultExecTimeout = 30 * time.Second +type ctxKey string + type Executor struct { tf *tfexec.Terraform timeout time.Duration diff --git a/internal/terraform/exec/exec_mock.go b/internal/terraform/exec/exec_mock.go index 4999efeb1..73f4173d3 100644 --- a/internal/terraform/exec/exec_mock.go +++ b/internal/terraform/exec/exec_mock.go @@ -1,12 +1,34 @@ package exec import ( + "context" + "fmt" + exec_mock "github.com/hashicorp/terraform-ls/internal/terraform/exec/mock" "github.com/stretchr/testify/mock" ) -func NewMockExecutor(calls []*mock.Call) ExecutorFactory { - return func(string, string) (TerraformExecutor, error) { +type TerraformMockCalls struct { + PerWorkDir map[string][]*mock.Call + AnyWorkDir []*mock.Call +} + +func NewMockExecutor(calls *TerraformMockCalls) ExecutorFactory { + return func(workDir string, execPath string) (TerraformExecutor, error) { + if calls == nil { + return nil, fmt.Errorf("%s: no mock calls defined", workDir) + } + mockCalls := calls.AnyWorkDir + if len(calls.PerWorkDir) > 0 { + mc, ok := calls.PerWorkDir[workDir] + if ok { + mockCalls = mc + } + } + if len(mockCalls) == 0 { + return nil, fmt.Errorf("%s: no mock calls available for this workdir", workDir) + } + me := &exec_mock.Executor{} firstCalls := []*mock.Call{ { @@ -15,7 +37,19 @@ func NewMockExecutor(calls []*mock.Call) ExecutorFactory { Repeatability: 1, }, } - me.ExpectedCalls = append(firstCalls, calls...) + + me.ExpectedCalls = append(firstCalls, mockCalls...) return me, nil } } + +var ctxExecutorFactory = ctxKey("executor factory") + +func ExecutorFactoryFromContext(ctx context.Context) (ExecutorFactory, bool) { + f, ok := ctx.Value(ctxExecutorFactory).(ExecutorFactory) + return f, ok +} + +func WithExecutorFactory(ctx context.Context, f ExecutorFactory) context.Context { + return context.WithValue(ctx, ctxExecutorFactory, f) +} diff --git a/internal/terraform/exec/exec_opts.go b/internal/terraform/exec/exec_opts.go new file mode 100644 index 000000000..37cae5267 --- /dev/null +++ b/internal/terraform/exec/exec_opts.go @@ -0,0 +1,23 @@ +package exec + +import ( + "context" + "time" +) + +type ExecutorOpts struct { + ExecPath string + ExecLogPath string + Timeout time.Duration +} + +var ctxExecOpts = ctxKey("executor opts") + +func ExecutorOptsFromContext(ctx context.Context) (*ExecutorOpts, bool) { + opts, ok := ctx.Value(ctxExecOpts).(*ExecutorOpts) + return opts, ok +} + +func WithExecutorOpts(ctx context.Context, opts *ExecutorOpts) context.Context { + return context.WithValue(ctx, ctxExecOpts, opts) +} diff --git a/internal/terraform/module/file.go b/internal/terraform/module/file.go deleted file mode 100644 index 388469d20..000000000 --- a/internal/terraform/module/file.go +++ /dev/null @@ -1,43 +0,0 @@ -package module - -import ( - "fmt" - "os" -) - -func findFile(paths []string) (File, error) { - var lf File - var err error - - for _, path := range paths { - lf, err = newFile(path) - if err == nil { - return lf, nil - } - if !os.IsNotExist(err) { - return nil, err - } - } - - return nil, err -} - -type file struct { - path string -} - -func (f *file) Path() string { - return f.path -} - -func newFile(path string) (File, error) { - fi, err := os.Stat(path) - if err != nil { - return nil, err - } - if fi.IsDir() { - return nil, fmt.Errorf("expected %s to be a file, not a dir", path) - } - - return &file{path: path}, nil -} diff --git a/internal/terraform/module/module.go b/internal/terraform/module/module.go index 0156f42e9..d0c771617 100644 --- a/internal/terraform/module/module.go +++ b/internal/terraform/module/module.go @@ -1,674 +1,233 @@ package module import ( - "context" - "errors" - "fmt" - "io/ioutil" "log" - "os" "path/filepath" - "strings" "sync" - "time" - "github.com/hashicorp/go-multierror" "github.com/hashicorp/go-version" - "github.com/hashicorp/hcl-lang/decoder" - "github.com/hashicorp/hcl-lang/schema" "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/hclsyntax" tfjson "github.com/hashicorp/terraform-json" "github.com/hashicorp/terraform-ls/internal/filesystem" - "github.com/hashicorp/terraform-ls/internal/schemas" - "github.com/hashicorp/terraform-ls/internal/terraform/discovery" - "github.com/hashicorp/terraform-ls/internal/terraform/exec" - tfschema "github.com/hashicorp/terraform-schema/schema" + "github.com/hashicorp/terraform-ls/internal/terraform/datadir" ) type module struct { path string + fs filesystem.Filesystem logger *log.Logger - // loading - isLoading bool - isLoadingMu *sync.RWMutex - loadingDone <-chan struct{} - cancelLoading context.CancelFunc - loadErr error - loadErrMu *sync.RWMutex - - // module cache - moduleMu *sync.RWMutex - moduleManifestFile File - moduleManifest *moduleManifest - - // plugin (provider schema) cache - pluginMu *sync.RWMutex - pluginLockFile File - providerSchema *tfjson.ProviderSchemas - providerSchemaMu *sync.RWMutex - providerVersions map[string]*version.Version - - // terraform executor - tfLoadingDone bool - tfLoadingMu *sync.RWMutex - tfExec exec.TerraformExecutor - tfNewExecutor exec.ExecutorFactory - tfExecPath string - tfExecTimeout time.Duration - tfExecLogPath string - - // terraform discovery - tfDiscoFunc discovery.DiscoveryFunc - tfDiscoErr error - tfVersion *version.Version - tfVersionErr error - - // core schema - coreSchema *schema.BodySchema - coreSchemaMu *sync.RWMutex - - // decoder - isParsed bool - isParsedMu *sync.RWMutex - pFilesMap map[string]*hcl.File - parsedDiags map[string]hcl.Diagnostics - parserMu *sync.RWMutex - filesystem filesystem.Filesystem + // module manifest + modManifest *datadir.ModuleManifest + modManifestErr error + modManifestMu *sync.RWMutex + modManigestState OpState + modManifestStateMu *sync.RWMutex + + // provider schema + providerSchema *tfjson.ProviderSchemas + providerSchemaErr error + providerSchemaMu *sync.RWMutex + providerSchemaState OpState + providerSchemaStateMu *sync.RWMutex + + // terraform exec path + tfExecPath string + tfExecPathMu *sync.RWMutex + + // terraform version + tfVersion *version.Version + tfVersionErr error + tfVersionMu *sync.RWMutex + tfVersionState OpState + tfVersionStateMu *sync.RWMutex + + // provider versions + providerVersions map[string]*version.Version + providerVersionsMu *sync.RWMutex + + // config (HCL) parser + parsedFiles map[string]*hcl.File + parsingErr error + parserMu *sync.RWMutex + parserState OpState + parserStateMu *sync.RWMutex + + // module diagnostics + diags map[string]hcl.Diagnostics + diagsMu *sync.RWMutex } func newModule(fs filesystem.Filesystem, dir string) *module { return &module{ - path: dir, - filesystem: fs, - logger: defaultLogger, - isLoadingMu: &sync.RWMutex{}, - loadErrMu: &sync.RWMutex{}, - moduleMu: &sync.RWMutex{}, - pluginMu: &sync.RWMutex{}, - providerSchemaMu: &sync.RWMutex{}, - tfLoadingMu: &sync.RWMutex{}, - coreSchema: tfschema.UniversalCoreModuleSchema(), - coreSchemaMu: &sync.RWMutex{}, - isParsedMu: &sync.RWMutex{}, - pFilesMap: make(map[string]*hcl.File, 0), - providerVersions: make(map[string]*version.Version, 0), - parserMu: &sync.RWMutex{}, - } -} - -var defaultLogger = log.New(ioutil.Discard, "", 0) - -func NewModule(ctx context.Context, fs filesystem.Filesystem, dir string) (Module, error) { - m := newModule(fs, dir) - - d := &discovery.Discovery{} - m.tfDiscoFunc = d.LookPath - - m.tfNewExecutor = exec.NewExecutor - - err := m.discoverCaches(ctx, dir) - if err != nil { - return m, err - } - - return m, m.load(ctx) -} - -func (m *module) discoverCaches(ctx context.Context, dir string) error { - var errs *multierror.Error - err := m.discoverPluginCache(dir) - if err != nil { - errs = multierror.Append(errs, err) - } - - err = m.discoverModuleCache(dir) - if err != nil { - errs = multierror.Append(errs, err) - } - - return errs.ErrorOrNil() -} - -func (m *module) WasInitialized() (bool, error) { - tfDirPath := filepath.Join(m.Path(), ".terraform") - - f, err := m.filesystem.Open(tfDirPath) - if err != nil { - return false, err - } - defer f.Close() - fi, err := f.Stat() - if err != nil { - return false, err - } - if !fi.IsDir() { - return false, fmt.Errorf("%s is not a directory", tfDirPath) - } - - return true, nil -} - -func (m *module) discoverPluginCache(dir string) error { - m.pluginMu.Lock() - defer m.pluginMu.Unlock() - - lockPaths := pluginLockFilePaths(dir) - lf, err := findFile(lockPaths) - if err != nil { - if os.IsNotExist(err) { - m.logger.Printf("no plugin cache found: %s", err.Error()) - return nil - } + path: dir, + fs: fs, + logger: defaultLogger, - return fmt.Errorf("unable to calculate hash: %w", err) + modManifestMu: &sync.RWMutex{}, + modManifestStateMu: &sync.RWMutex{}, + providerSchemaMu: &sync.RWMutex{}, + providerSchemaStateMu: &sync.RWMutex{}, + providerVersions: make(map[string]*version.Version, 0), + providerVersionsMu: &sync.RWMutex{}, + tfVersionMu: &sync.RWMutex{}, + tfVersionStateMu: &sync.RWMutex{}, + tfExecPathMu: &sync.RWMutex{}, + parsedFiles: make(map[string]*hcl.File, 0), + parserMu: &sync.RWMutex{}, + parserStateMu: &sync.RWMutex{}, + diagsMu: &sync.RWMutex{}, } - m.pluginLockFile = lf - return nil } -func (m *module) discoverModuleCache(dir string) error { - m.moduleMu.Lock() - defer m.moduleMu.Unlock() - - lf, err := newFile(moduleManifestFilePath(dir)) - if err != nil { - if os.IsNotExist(err) { - m.logger.Printf("no module manifest file found: %s", err.Error()) - return nil - } - - return fmt.Errorf("unable to calculate hash: %w", err) - } - m.moduleManifestFile = lf - return nil -} - -func (m *module) Modules() []ModuleRecord { - m.moduleMu.Lock() - defer m.moduleMu.Unlock() - if m.moduleManifest == nil { - return []ModuleRecord{} - } - - return m.moduleManifest.Records -} - -func (m *module) SetLogger(logger *log.Logger) { - m.logger = logger -} - -func (m *module) StartLoading() error { - if !m.IsLoadingDone() { - return fmt.Errorf("module is already being loaded") - } - ctx, cancelFunc := context.WithCancel(context.Background()) - m.cancelLoading = cancelFunc - m.loadingDone = ctx.Done() - - go func(ctx context.Context) { - m.setLoadErr(m.load(ctx)) - }(ctx) - return nil +func NewModule(fs filesystem.Filesystem, dir string) Module { + return newModule(fs, dir) } -func (m *module) CancelLoading() { - if !m.IsLoadingDone() && m.cancelLoading != nil { - m.cancelLoading() - } - m.setLoadingState(false) -} - -func (m *module) LoadingDone() <-chan struct{} { - return m.loadingDone -} - -func (m *module) load(ctx context.Context) error { - var errs *multierror.Error - defer m.CancelLoading() - - // reset internal loading state - m.setLoadingState(true) - - // The following operations have to happen in a particular order - // as they depend on the internal state as mutated by each operation - - err := m.UpdateModuleManifest(m.moduleManifestFile) - errs = multierror.Append(errs, err) - - err = m.discoverTerraformExecutor(ctx) - m.tfDiscoErr = err - errs = multierror.Append(errs, err) - - err = m.discoverTerraformVersion(ctx) - m.tfVersionErr = err - errs = multierror.Append(errs, err) - - err = m.findAndSetCoreSchema() +func (m *module) HasOpenFiles() bool { + openFiles, err := m.fs.HasOpenFiles(m.Path()) if err != nil { - m.logger.Printf("%s: %s - falling back to universal schema", + m.logger.Printf("%s: failed to check whether module has open files: %s", m.Path(), err) } - - err = m.UpdateProviderSchemaCache(ctx, m.pluginLockFile) - errs = multierror.Append(errs, err) - - m.logger.Printf("loading of module %s finished: %s", - m.Path(), errs) - return errs.ErrorOrNil() -} - -func (m *module) setLoadingState(isLoading bool) { - m.isLoadingMu.Lock() - defer m.isLoadingMu.Unlock() - m.isLoading = isLoading + return openFiles } -func (m *module) IsLoadingDone() bool { - m.isLoadingMu.RLock() - defer m.isLoadingMu.RUnlock() - return !m.isLoading -} - -func (m *module) discoverTerraformExecutor(ctx context.Context) error { - defer func() { - m.setTfDiscoveryFinished(true) - }() - - tfPath := m.tfExecPath - if tfPath == "" { - var err error - tfPath, err = m.tfDiscoFunc() - if err != nil { - return err - } - } - - tf, err := m.tfNewExecutor(m.path, tfPath) - if err != nil { - return err - } - - tf.SetLogger(m.logger) - - if m.tfExecLogPath != "" { - tf.SetExecLogPath(m.tfExecLogPath) - } - - if m.tfExecTimeout != 0 { - tf.SetTimeout(m.tfExecTimeout) - } - - m.tfExec = tf - - return nil -} - -func (m *module) ExecuteTerraformInit(ctx context.Context) error { - if !m.IsTerraformAvailable() { - if err := m.discoverTerraformExecutor(ctx); err != nil { - return err - } - } - - return m.tfExec.Init(ctx) -} - -func (m *module) ExecuteTerraformValidate(ctx context.Context) (map[string]hcl.Diagnostics, error) { - diagsMap := make(map[string]hcl.Diagnostics) - - if !m.IsTerraformAvailable() { - if err := m.discoverTerraformExecutor(ctx); err != nil { - return diagsMap, err - } - } - - if !m.IsParsed() { - if err := m.ParseFiles(); err != nil { - return diagsMap, err - } - } - - // an entry for each file should exist, even if there are no diags - for filename := range m.parsedFiles() { - diagsMap[filename] = make(hcl.Diagnostics, 0) - } - // since validation applies to linked modules, create an entry for all - // files of linked modules - for _, mod := range m.moduleManifest.Records { - if mod.IsRoot() { - // skip root module - continue - } - if mod.IsExternal() { - // skip external module - continue - } - - absPath := filepath.Join(m.moduleManifest.rootDir, mod.Dir) - infos, err := m.filesystem.ReadDir(absPath) - if err != nil { - return diagsMap, fmt.Errorf("failed to read module at %q: %w", absPath, err) - } - - for _, info := range infos { - if info.IsDir() { - // We only care about files - continue - } - - name := info.Name() - if !strings.HasSuffix(name, ".tf") || IsIgnoredFile(name) { - continue - } - - // map entries are relative to the parent module path - filename := filepath.Join(mod.Dir, name) - - diagsMap[filename] = make(hcl.Diagnostics, 0) - } - } - - validationDiags, err := m.tfExec.Validate(ctx) - if err != nil { - return diagsMap, err - } - - // tfjson.Diagnostic is a conversion of an internal diag to terraform core, - // tfdiags, which is effectively based on hcl.Diagnostic. - // This process is really just converting it back to hcl.Diagnotic - // since it is the defacto diagnostic type for our codebase currently - // https://github.com/hashicorp/terraform/blob/ae025248cc0712bf53c675dc2fe77af4276dd5cc/command/validate.go#L138 - for _, d := range validationDiags { - // the diagnostic must be tied to a file to exist in the map - if d.Range == nil || d.Range.Filename == "" { - continue - } - - diags := diagsMap[d.Range.Filename] - - var severity hcl.DiagnosticSeverity - if d.Severity == "error" { - severity = hcl.DiagError - } else if d.Severity == "warning" { - severity = hcl.DiagWarning - } - - diags = append(diags, &hcl.Diagnostic{ - Severity: severity, - Summary: d.Summary, - Detail: d.Detail, - Subject: &hcl.Range{ - Filename: d.Range.Filename, - Start: hcl.Pos(d.Range.Start), - End: hcl.Pos(d.Range.End), - }, - }) - diagsMap[d.Range.Filename] = diags - } - - return diagsMap, nil +func (m *module) SetTerraformVersion(v *version.Version, err error) { + m.tfVersionMu.Lock() + defer m.tfVersionMu.Unlock() + m.tfVersion = v + m.tfVersionErr = err } -func (m *module) discoverTerraformVersion(ctx context.Context) error { - if m.tfExec == nil { - return errors.New("no terraform executor - unable to read version") - } - - version, providerVersions, err := m.tfExec.Version(ctx) - if err != nil { - return err - } - m.logger.Printf("Terraform version %s found at %s for %s", version, - m.tfExec.GetExecPath(), m.Path()) - m.tfVersion = version - - m.providerVersions = providerVersions - - return nil +func (m *module) TerraformVersion() (*version.Version, error) { + m.tfVersionMu.RLock() + defer m.tfVersionMu.RUnlock() + return m.tfVersion, m.tfVersionErr } -func (m *module) findAndSetCoreSchema() error { - if m.tfVersion == nil { - return errors.New("unable to find core schema without version") - } - - coreSchema, err := tfschema.CoreModuleSchemaForVersion(m.tfVersion) - if err != nil { - return err - } - - m.coreSchemaMu.Lock() - m.coreSchema = coreSchema - m.coreSchemaMu.Unlock() - - return nil +func (m *module) SetProviderVersions(pv map[string]*version.Version) { + m.providerVersionsMu.Lock() + defer m.providerVersionsMu.Unlock() + m.providerVersions = pv } -func (m *module) LoadError() error { - m.loadErrMu.RLock() - defer m.loadErrMu.RUnlock() - return m.loadErr +func (m *module) ProviderVersions() map[string]*version.Version { + m.providerVersionsMu.RLock() + defer m.providerVersionsMu.RUnlock() + return m.providerVersions } -func (m *module) setLoadErr(err error) { - m.loadErrMu.Lock() - defer m.loadErrMu.Unlock() - m.loadErr = err +func (m *module) TerraformVersionState() OpState { + m.tfVersionMu.RLock() + defer m.tfVersionMu.RUnlock() + return m.tfVersionState } -func (m *module) Path() string { - return m.path +func (m *module) SetTerraformVersionState(state OpState) { + m.tfVersionMu.Lock() + defer m.tfVersionMu.Unlock() + m.tfVersionState = state } -func (m *module) MatchesPath(path string) bool { - return filepath.Clean(m.path) == filepath.Clean(path) +func (m *module) SetModuleManifest(manifest *datadir.ModuleManifest, err error) { + m.modManifestMu.Lock() + defer m.modManifestMu.Unlock() + m.modManifest = manifest + m.modManifestErr = err } -// HumanReadablePath helps display shorter, but still relevant paths -func (m *module) HumanReadablePath(rootDir string) string { - if rootDir == "" { - return m.path - } - - // absolute paths can be too long for UI/messages, - // so we just display relative to root dir - relDir, err := filepath.Rel(rootDir, m.path) - if err != nil { - return m.path - } - - if relDir == "." { - // Name of the root dir is more helpful than "." - return filepath.Base(rootDir) - } - - return relDir +func (m *module) ModuleManifestState() OpState { + m.modManifestMu.RLock() + defer m.modManifestMu.RUnlock() + return m.modManigestState } -func (m *module) UpdateModuleManifest(lockFile File) error { - m.moduleMu.Lock() - defer m.moduleMu.Unlock() - - if lockFile == nil { - m.logger.Printf("ignoring module update as no lock file was found for %s", m.Path()) - return nil - } - - m.moduleManifestFile = lockFile - - mm, err := ParseModuleManifestFromFile(lockFile.Path()) - if err != nil { - return fmt.Errorf("failed to update module manifest: %w", err) - } - - m.moduleManifest = mm - m.logger.Printf("updated module manifest - %d references parsed for %s", - len(mm.Records), m.Path()) - return nil +func (m *module) SetModuleManifestParsingState(state OpState) { + m.modManifestMu.Lock() + defer m.modManifestMu.Unlock() + m.modManigestState = state } -func (m *module) DecoderWithSchema(schema *schema.BodySchema) (*decoder.Decoder, error) { - d, err := m.Decoder() - if err != nil { - return nil, err - } - - d.SetSchema(schema) - - return d, nil +func (m *module) SetProviderSchemas(ps *tfjson.ProviderSchemas, err error) { + m.providerSchemaMu.Lock() + defer m.providerSchemaMu.Unlock() + m.providerSchema = ps + m.providerSchemaErr = err } -func (m *module) Decoder() (*decoder.Decoder, error) { - d := decoder.NewDecoder() - - for name, f := range m.parsedFiles() { - err := d.LoadFile(name, f) - if err != nil { - return nil, fmt.Errorf("failed to load a file: %w", err) - } - } - return d, nil +func (m *module) ProviderSchema() (*tfjson.ProviderSchemas, error) { + m.providerSchemaMu.RLock() + defer m.providerSchemaMu.RUnlock() + return m.providerSchema, m.providerSchemaErr } -func (m *module) IsProviderSchemaLoaded() bool { +func (m *module) ProviderSchemaState() OpState { m.providerSchemaMu.RLock() defer m.providerSchemaMu.RUnlock() - return m.providerSchema != nil + return m.providerSchemaState } -func (m *module) IsParsed() bool { - m.isParsedMu.RLock() - defer m.isParsedMu.RUnlock() - return m.isParsed +func (m *module) SetProviderSchemaObtainingState(state OpState) { + m.providerSchemaMu.Lock() + defer m.providerSchemaMu.Unlock() + m.providerSchemaState = state } -func (m *module) setIsParsed(parsed bool) { - m.isParsedMu.Lock() - defer m.isParsedMu.Unlock() - m.isParsed = parsed +func (m *module) ParsedFiles() (map[string]*hcl.File, error) { + m.parserMu.RLock() + defer m.parserMu.RUnlock() + return m.parsedFiles, m.parsingErr } -func (m *module) ParseFiles() error { +func (m *module) SetParsedFiles(files map[string]*hcl.File, err error) { m.parserMu.Lock() defer m.parserMu.Unlock() - - files := make(map[string]*hcl.File, 0) - diags := make(map[string]hcl.Diagnostics, 0) - - infos, err := m.filesystem.ReadDir(m.Path()) - if err != nil { - return fmt.Errorf("failed to read module at %q: %w", m.Path(), err) - } - - for _, info := range infos { - if info.IsDir() { - // We only care about files - continue - } - - name := info.Name() - if !strings.HasSuffix(name, ".tf") || IsIgnoredFile(name) { - continue - } - - // TODO: overrides - - fullPath := filepath.Join(m.Path(), name) - - src, err := m.filesystem.ReadFile(fullPath) - if err != nil { - return fmt.Errorf("failed to read %q: %s", name, err) - } - - m.logger.Printf("parsing file %q", name) - f, pDiags := hclsyntax.ParseConfig(src, name, hcl.InitialPos) - diags[name] = pDiags - if f != nil { - files[name] = f - } - } - - m.pFilesMap = files - m.parsedDiags = diags - m.setIsParsed(true) - - return nil + m.parsedFiles = files + m.parsingErr = err } -func (m *module) ParsedDiagnostics() map[string]hcl.Diagnostics { - m.parserMu.Lock() - defer m.parserMu.Unlock() - return m.parsedDiags +func (m *module) SetDiagnostics(diags map[string]hcl.Diagnostics) { + m.diagsMu.Lock() + defer m.diagsMu.Unlock() + m.diags = diags } -func (m *module) parsedFiles() map[string]*hcl.File { +func (m *module) ConfigParsingState() OpState { m.parserMu.RLock() defer m.parserMu.RUnlock() - - return m.pFilesMap + return m.parserState } -func (m *module) MergedSchema() (*schema.BodySchema, error) { - m.coreSchemaMu.RLock() - defer m.coreSchemaMu.RUnlock() - - if !m.IsParsed() { - err := m.ParseFiles() - if err != nil { - return nil, err - } - } - - ps, vOut, err := schemas.PreloadedProviderSchemas() - if err != nil { - return nil, err - } - providerVersions := vOut.Providers - tfVersion := vOut.Core - - if m.IsProviderSchemaLoaded() { - m.providerSchemaMu.RLock() - defer m.providerSchemaMu.RUnlock() - ps = m.providerSchema - providerVersions = m.providerVersions - tfVersion = m.tfVersion - } - - if ps == nil { - m.logger.Print("provider schemas is nil... skipping merge with core schema") - return m.coreSchema, nil - } - - sm := tfschema.NewSchemaMerger(m.coreSchema) - sm.SetCoreVersion(tfVersion) - sm.SetParsedFiles(m.parsedFiles()) - - err = sm.SetProviderVersions(providerVersions) - if err != nil { - return nil, err - } +func (m *module) SetConfigParsingState(state OpState) { + m.parserMu.Lock() + defer m.parserMu.Unlock() + m.parserState = state +} - return sm.MergeWithJsonProviderSchemas(ps) +func (m *module) ModuleManifest() (*datadir.ModuleManifest, error) { + m.modManifestMu.RLock() + defer m.modManifestMu.RUnlock() + return m.modManifest, m.modManifestErr } -// IsIgnoredFile returns true if the given filename (which must not have a -// directory path ahead of it) should be ignored as e.g. an editor swap file. -func IsIgnoredFile(name string) bool { - return strings.HasPrefix(name, ".") || // Unix-like hidden files - strings.HasSuffix(name, "~") || // vim - strings.HasPrefix(name, "#") && strings.HasSuffix(name, "#") // emacs +func (m *module) ModuleCalls() []datadir.ModuleRecord { + m.modManifestMu.RLock() + defer m.modManifestMu.RUnlock() + if m.modManifest == nil { + return []datadir.ModuleRecord{} + } + return m.modManifest.Records } -func (m *module) ReferencesModulePath(path string) bool { - m.moduleMu.Lock() - defer m.moduleMu.Unlock() - if m.moduleManifest == nil { +func (m *module) CallsModule(path string) bool { + m.modManifestMu.RLock() + defer m.modManifestMu.RUnlock() + if m.modManifest == nil { return false } - for _, mod := range m.moduleManifest.Records { + for _, mod := range m.modManifest.Records { if mod.IsRoot() { // skip root module, as that's tracked separately continue @@ -677,7 +236,7 @@ func (m *module) ReferencesModulePath(path string) bool { // skip external modules as these shouldn't be modified from cache continue } - absPath := filepath.Join(m.moduleManifest.rootDir, mod.Dir) + absPath := filepath.Join(m.modManifest.RootDir(), mod.Dir) if pathEquals(absPath, path) { return true } @@ -686,97 +245,47 @@ func (m *module) ReferencesModulePath(path string) bool { return false } -func (m *module) TerraformFormatter() (exec.Formatter, error) { - if !m.HasTerraformDiscoveryFinished() { - return nil, fmt.Errorf("terraform is not loaded yet") - } - - if !m.IsTerraformAvailable() { - return nil, fmt.Errorf("terraform is not available") - } - - return m.tfExec.Format, nil -} - -func (m *module) HasTerraformDiscoveryFinished() bool { - m.tfLoadingMu.RLock() - defer m.tfLoadingMu.RUnlock() - return m.tfLoadingDone +func (m *module) SetLogger(logger *log.Logger) { + m.logger = logger } -func (m *module) setTfDiscoveryFinished(isLoaded bool) { - m.tfLoadingMu.Lock() - defer m.tfLoadingMu.Unlock() - m.tfLoadingDone = isLoaded +func (m *module) Path() string { + return m.path } -func (m *module) IsTerraformAvailable() bool { - return m.HasTerraformDiscoveryFinished() && m.tfExec != nil +func (m *module) MatchesPath(path string) bool { + return pathEquals(m.path, path) } -func (m *module) UpdateProviderSchemaCache(ctx context.Context, lockFile File) error { - m.pluginMu.Lock() - defer m.pluginMu.Unlock() - - if !m.IsTerraformAvailable() { - return fmt.Errorf("cannot update provider schema as terraform is unavailable") - } - - if lockFile == nil { - m.logger.Printf("ignoring provider schema update as no lock file was provided for %s", - m.Path()) - return nil +// HumanReadablePath helps display shorter, but still relevant paths +func (m *module) HumanReadablePath(rootDir string) string { + if rootDir == "" { + return m.path } - m.pluginLockFile = lockFile - - schemas, err := m.tfExec.ProviderSchemas(ctx) + // absolute paths can be too long for UI/messages, + // so we just display relative to root dir + relDir, err := filepath.Rel(rootDir, m.path) if err != nil { - return err + return m.path } - m.providerSchemaMu.Lock() - m.providerSchema = schemas - m.providerSchemaMu.Unlock() - - return nil -} - -func (m *module) PathsToWatch() []string { - m.pluginMu.RLock() - m.moduleMu.RLock() - defer m.moduleMu.RUnlock() - defer m.pluginMu.RUnlock() - - files := make([]string, 0) - if m.pluginLockFile != nil { - files = append(files, m.pluginLockFile.Path()) - } - if m.moduleManifestFile != nil { - files = append(files, m.moduleManifestFile.Path()) + if relDir == "." { + // Name of the root dir is more helpful than "." + return filepath.Base(rootDir) } - return files + return relDir } -func (m *module) IsKnownModuleManifestFile(path string) bool { - m.moduleMu.RLock() - defer m.moduleMu.RUnlock() - - if m.moduleManifestFile == nil { - return false - } - - return pathEquals(m.moduleManifestFile.Path(), path) +func (m *module) TerraformExecPath() string { + m.tfExecPathMu.RLock() + defer m.tfExecPathMu.RUnlock() + return m.tfExecPath } -func (m *module) IsKnownPluginLockFile(path string) bool { - m.pluginMu.RLock() - defer m.pluginMu.RUnlock() - - if m.pluginLockFile == nil { - return false - } - - return pathEquals(m.pluginLockFile.Path(), path) +func (m *module) Diagnostics() map[string]hcl.Diagnostics { + m.diagsMu.RLock() + defer m.diagsMu.RUnlock() + return m.diags } diff --git a/internal/terraform/module/module_loader.go b/internal/terraform/module/module_loader.go new file mode 100644 index 000000000..46fada294 --- /dev/null +++ b/internal/terraform/module/module_loader.go @@ -0,0 +1,163 @@ +package module + +import ( + "context" + "log" + "runtime" + "sync/atomic" + + "github.com/hashicorp/terraform-ls/internal/terraform/exec" +) + +type moduleLoader struct { + queue moduleOpsQueue + nonPrioParallelism int64 + prioParallelism int64 + logger *log.Logger + tfExecOpts *exec.ExecutorOpts + opsToDispatch chan ModuleOperation + + loadingCount *int64 + prioLoadingCount *int64 +} + +func newModuleLoader() *moduleLoader { + nonPrioParallelism := 2 * runtime.NumCPU() + prioParallelism := 1 * runtime.NumCPU() + + plc, lc := int64(0), int64(0) + ml := &moduleLoader{ + queue: newModuleOpsQueue(), + logger: defaultLogger, + nonPrioParallelism: int64(nonPrioParallelism), + prioParallelism: int64(prioParallelism), + opsToDispatch: make(chan ModuleOperation, 1), + loadingCount: &lc, + prioLoadingCount: &plc, + } + + return ml +} + +func (ml *moduleLoader) SetLogger(logger *log.Logger) { + ml.logger = logger +} + +func (ml *moduleLoader) Start(ctx context.Context) { + go ml.run(ctx) +} + +func (ml *moduleLoader) run(ctx context.Context) { + for { + select { + case <-ctx.Done(): + ml.logger.Println("Cancelling module loader...") + return + case nextOp, ok := <-ml.opsToDispatch: + if !ok { + ml.logger.Println("Failed to get next operation") + return + } + + if nextOp.Module.HasOpenFiles() && ml.prioCapacity() > 0 { + atomic.AddInt64(ml.prioLoadingCount, 1) + mod := ml.queue.PopOp() + go func(ml *moduleLoader) { + defer atomic.AddInt64(ml.prioLoadingCount, -1) + ml.executeModuleOp(ctx, mod) + }(ml) + } else if ml.nonPrioCapacity() > 0 { + atomic.AddInt64(ml.loadingCount, 1) + mod := ml.queue.PopOp() + go func(ml *moduleLoader) { + defer atomic.AddInt64(ml.loadingCount, -1) + ml.executeModuleOp(ctx, mod) + }(ml) + } + } + } +} + +func (ml *moduleLoader) tryDispatchingModuleOp() { + totalCapacity := ml.nonPrioCapacity() + ml.prioCapacity() + opsInQueue := ml.queue.Len() + + // Keep scheduling work from queue if we have capacity + if opsInQueue > 0 && totalCapacity > 0 { + item := ml.queue.Peek() + nextModOp := item.(ModuleOperation) + ml.opsToDispatch <- nextModOp + } +} + +func (ml *moduleLoader) prioCapacity() int64 { + return ml.prioParallelism - atomic.LoadInt64(ml.prioLoadingCount) +} + +func (ml *moduleLoader) nonPrioCapacity() int64 { + return ml.prioParallelism - atomic.LoadInt64(ml.loadingCount) +} + +func (ml *moduleLoader) executeModuleOp(ctx context.Context, modOp ModuleOperation) { + ml.logger.Printf("executing %q for %s", modOp.Type, modOp.Module.Path()) + // TODO: Report progress in % for each op based on queue length + defer ml.logger.Printf("finished %q for %s", modOp.Type, modOp.Module.Path()) + defer modOp.markAsDone() + defer ml.tryDispatchingModuleOp() + + switch modOp.Type { + case OpTypeGetTerraformVersion: + GetTerraformVersion(ctx, modOp.Module) + return + case OpTypeObtainSchema: + ObtainSchema(ctx, modOp.Module) + return + case OpTypeParseConfiguration: + ParseConfiguration(modOp.Module) + return + case OpTypeParseModuleManifest: + ParseModuleManifest(modOp.Module) + return + } + + ml.logger.Printf("%s: unknown operation (%#v) for module operation", + modOp.Module.Path(), modOp.Type) +} + +func (ml *moduleLoader) EnqueueModuleOp(modOp ModuleOperation) { + m := modOp.Module + mod := m.(*module) + + ml.logger.Printf("ML: enqueing %q module operation: %s", modOp.Type, mod.Path()) + + switch modOp.Type { + case OpTypeGetTerraformVersion: + if mod.TerraformVersionState() == OpStateQueued { + // avoid enqueuing duplicate operation + return + } + mod.SetTerraformVersionState(OpStateQueued) + case OpTypeObtainSchema: + if mod.ProviderSchemaState() == OpStateQueued { + // avoid enqueuing duplicate operation + return + } + mod.SetProviderSchemaObtainingState(OpStateQueued) + case OpTypeParseConfiguration: + if mod.ConfigParsingState() == OpStateQueued { + // avoid enqueuing duplicate operation + return + } + mod.SetConfigParsingState(OpStateQueued) + case OpTypeParseModuleManifest: + if mod.ModuleManifestState() == OpStateQueued { + // avoid enqueuing duplicate operation + return + } + mod.SetModuleManifestParsingState(OpStateQueued) + } + + ml.queue.PushOp(modOp) + + ml.tryDispatchingModuleOp() +} diff --git a/internal/terraform/module/module_manager.go b/internal/terraform/module/module_manager.go index d37f81bdc..c0920bba6 100644 --- a/internal/terraform/module/module_manager.go +++ b/internal/terraform/module/module_manager.go @@ -4,354 +4,238 @@ import ( "context" "fmt" "log" - "os" "path/filepath" - "runtime" - "strings" - "time" - "github.com/gammazero/workerpool" + "github.com/hashicorp/go-version" "github.com/hashicorp/hcl-lang/schema" + tfjson "github.com/hashicorp/terraform-json" "github.com/hashicorp/terraform-ls/internal/filesystem" - "github.com/hashicorp/terraform-ls/internal/terraform/discovery" - "github.com/hashicorp/terraform-ls/internal/terraform/exec" + "github.com/hashicorp/terraform-ls/internal/schemas" + tfschema "github.com/hashicorp/terraform-schema/schema" ) type moduleManager struct { - modules []*module - newModule ModuleFactory - filesystem filesystem.Filesystem + modules []*module + fs filesystem.Filesystem + loader *moduleLoader syncLoading bool - workerPool *workerpool.WorkerPool + cancelFunc context.CancelFunc logger *log.Logger - - // terraform discovery - tfDiscoFunc discovery.DiscoveryFunc - - // terraform executor - tfNewExecutor exec.ExecutorFactory - tfExecPath string - tfExecTimeout time.Duration - tfExecLogPath string } -func NewModuleManager(fs filesystem.Filesystem) ModuleManager { - return newModuleManager(fs) -} - -func newModuleManager(fs filesystem.Filesystem) *moduleManager { - d := &discovery.Discovery{} +func NewModuleManager(ctx context.Context, fs filesystem.Filesystem) ModuleManager { + mm := newModuleManager(fs) - defaultSize := 3 * runtime.NumCPU() - wp := workerpool.New(defaultSize) + ctx, cancelFunc := context.WithCancel(ctx) + mm.cancelFunc = cancelFunc + mm.loader.Start(ctx) - mm := &moduleManager{ - modules: make([]*module, 0), - filesystem: fs, - workerPool: wp, - logger: defaultLogger, - tfDiscoFunc: d.LookPath, - tfNewExecutor: exec.NewExecutor, - } - mm.newModule = mm.defaultModuleFactory return mm } -func (mm *moduleManager) WorkerPoolSize() int { - return mm.workerPool.Size() -} - -func (mm *moduleManager) WorkerQueueSize() int { - return mm.workerPool.WaitingQueueSize() -} - -func (mm *moduleManager) defaultModuleFactory(ctx context.Context, dir string) (*module, error) { - mod := newModule(mm.filesystem, dir) +func NewSyncModuleManager(ctx context.Context, fs filesystem.Filesystem) ModuleManager { + mm := newModuleManager(fs) - mod.SetLogger(mm.logger) + ctx, cancelFunc := context.WithCancel(ctx) + mm.cancelFunc = cancelFunc + mm.syncLoading = true - d := &discovery.Discovery{} - mod.tfDiscoFunc = d.LookPath - mod.tfNewExecutor = exec.NewExecutor + mm.loader.Start(ctx) - mod.tfExecPath = mm.tfExecPath - mod.tfExecTimeout = mm.tfExecTimeout - mod.tfExecLogPath = mm.tfExecLogPath - - return mod, mod.discoverCaches(ctx, dir) -} - -func (mm *moduleManager) SetTerraformExecPath(path string) { - mm.tfExecPath = path -} - -func (mm *moduleManager) SetTerraformExecLogPath(logPath string) { - mm.tfExecLogPath = logPath + return mm } -func (mm *moduleManager) SetTerraformExecTimeout(timeout time.Duration) { - mm.tfExecTimeout = timeout +func newModuleManager(fs filesystem.Filesystem) *moduleManager { + mm := &moduleManager{ + modules: make([]*module, 0), + fs: fs, + logger: defaultLogger, + loader: newModuleLoader(), + } + return mm } func (mm *moduleManager) SetLogger(logger *log.Logger) { mm.logger = logger + mm.loader.SetLogger(logger) } -func (mm *moduleManager) InitAndUpdateModule(ctx context.Context, dir string) (Module, error) { - mod, err := mm.ModuleByPath(dir) - if err != nil { - return nil, fmt.Errorf("failed to get module: %+v", err) - } - - if err := mod.ExecuteTerraformInit(ctx); err != nil { - return nil, fmt.Errorf("failed to init module: %+v", err) - } - - m := mod.(*module) - m.discoverCaches(ctx, dir) - return mod, m.UpdateProviderSchemaCache(ctx, m.pluginLockFile) -} - -func (mm *moduleManager) AddAndStartLoadingModule(ctx context.Context, dir string) (Module, error) { - dir = filepath.Clean(dir) +func (mm *moduleManager) AddModule(modPath string) (Module, error) { + modPath = filepath.Clean(modPath) + mm.logger.Printf("MM: adding new module: %s", modPath) // TODO: Follow symlinks (requires proper test data) - if _, ok := mm.moduleByPath(dir); ok { - return nil, fmt.Errorf("module %s was already added", dir) + if _, ok := mm.moduleByPath(modPath); ok { + return nil, fmt.Errorf("module %s was already added", modPath) } - mod, err := mm.newModule(context.Background(), dir) - if err != nil { - return nil, err - } + mod := newModule(mm.fs, modPath) + mod.SetLogger(mm.logger) mm.modules = append(mm.modules, mod) - if mm.syncLoading { - mm.logger.Printf("synchronously loading module %s", dir) - return mod, mod.load(ctx) - } - - mm.logger.Printf("asynchronously loading module %s", dir) - mm.workerPool.Submit(func() { - mod := mod - err := mod.load(context.Background()) - mod.setLoadErr(err) - }) - return mod, nil } -func (mm *moduleManager) SchemaForPath(path string) (*schema.BodySchema, error) { - candidates := mm.ModuleCandidatesByPath(path) - for _, mod := range candidates { - schema, err := mod.MergedSchema() - if err != nil { - mm.logger.Printf("failed to merge schema for %s: %s", mod.Path(), err) - continue - } - if schema != nil { - mm.logger.Printf("found schema for %s at %s", path, mod.Path()) - return schema, nil - } - } - - mod, err := mm.ModuleByPath(path) +func (mm *moduleManager) EnqueueModuleOpWait(modPath string, opType OpType) error { + mod, err := mm.ModuleByPath(modPath) if err != nil { - return nil, err + return err } + modOp := NewModuleOperation(mod, opType) + mm.loader.EnqueueModuleOp(modOp) - return mod.MergedSchema() -} + <-modOp.Done() -func (mm *moduleManager) moduleByPath(dir string) (*module, bool) { - for _, mod := range mm.modules { - if pathEquals(mod.Path(), dir) { - return mod, true - } - } - return nil, false + return nil } -// ModuleCandidatesByPath finds any initialized modules -func (mm *moduleManager) ModuleCandidatesByPath(path string) Modules { - path = filepath.Clean(path) - - candidates := make([]Module, 0) - - // TODO: Follow symlinks (requires proper test data) - - mod, foundPath := mm.moduleByPath(path) - if foundPath { - inited, _ := mod.WasInitialized() - if inited { - candidates = append(candidates, mod) - } +func (mm *moduleManager) EnqueueModuleOp(modPath string, opType OpType) error { + mod, err := mm.ModuleByPath(modPath) + if err != nil { + return err } - if !foundPath { - dir := trimLockFilePath(path) - mod, ok := mm.moduleByPath(dir) - if ok { - inited, _ := mod.WasInitialized() - if inited { - candidates = append(candidates, mod) - } - } + modOp := NewModuleOperation(mod, opType) + mm.loader.EnqueueModuleOp(modOp) + if mm.syncLoading { + <-modOp.Done() } + return nil +} - for _, mod := range mm.modules { - if mod.ReferencesModulePath(path) { - candidates = append(candidates, mod) - } +func (mm *moduleManager) SchemaForModule(modPath string) (*schema.BodySchema, error) { + sources, err := mm.SchemaSourcesForModule(modPath) + if err != nil { + return nil, err } - return candidates -} + var ( + tfVersion *version.Version + coreSchema *schema.BodySchema + providerSchema *tfjson.ProviderSchemas + providerVersions map[string]*version.Version + ) -func (mm *moduleManager) ListModules() Modules { - modules := make([]Module, 0) - for _, mod := range mm.modules { - modules = append(modules, mod) + if len(sources) > 0 { + ps, err := sources[0].ProviderSchema() + if err == nil { + providerSchema = ps + } + providerVersions = sources[0].ProviderVersions() } - return modules -} -func (mm *moduleManager) ModuleByPath(path string) (Module, error) { - path = filepath.Clean(path) - if mod, ok := mm.moduleByPath(path); ok { - return mod, nil + mod, err := mm.ModuleByPath(modPath) + if err != nil { + return nil, err } - dir := trimLockFilePath(path) - - if mod, ok := mm.moduleByPath(dir); ok { - return mod, nil + if v, err := mod.TerraformVersion(); err == nil { + tfVersion = v } - return nil, &ModuleNotFoundErr{path} -} - -func (mm *moduleManager) IsProviderSchemaLoaded(path string) (bool, error) { - mod, err := mm.ModuleByPath(path) - if err != nil { - return false, err - } + if len(sources) == 0 { + mm.logger.Printf("falling back to preloaded schema for %s...", modPath) + ps, vOut, err := schemas.PreloadedProviderSchemas() + if err != nil { + return nil, err + } + if ps != nil { + providerSchema = ps + providerVersions = vOut.Providers - return mod.IsProviderSchemaLoaded(), nil -} + mm.logger.Printf("preloaded provider schema (%d providers) set for %s", + len(ps.Schemas), modPath) -func (mm *moduleManager) TerraformFormatterForDir(ctx context.Context, path string) (exec.Formatter, error) { - mod, err := mm.ModuleByPath(path) - if err != nil { - if IsModuleNotFound(err) { - return mm.newTerraformFormatter(ctx, path) + if tfVersion == nil { + tfVersion = vOut.Core + } } - return nil, err } - return mod.TerraformFormatter() -} - -func (mm *moduleManager) newTerraformFormatter(ctx context.Context, workDir string) (exec.Formatter, error) { - tfPath := mm.tfExecPath - if tfPath == "" { - var err error - tfPath, err = mm.tfDiscoFunc() + if tfVersion != nil { + coreSchema, err = tfschema.CoreModuleSchemaForVersion(tfVersion) if err != nil { return nil, err } + } else { + coreSchema = tfschema.UniversalCoreModuleSchema() } - tf, err := mm.tfNewExecutor(workDir, tfPath) - if err != nil { - return nil, err - } - - tf.SetLogger(mm.logger) - - if mm.tfExecLogPath != "" { - tf.SetExecLogPath(mm.tfExecLogPath) + merger := tfschema.NewSchemaMerger(coreSchema) + if tfVersion != nil { + merger.SetCoreVersion(tfVersion) } - - if mm.tfExecTimeout != 0 { - tf.SetTimeout(mm.tfExecTimeout) + if len(providerVersions) > 0 { + err = merger.SetProviderVersions(providerVersions) + if err != nil { + return nil, err + } } - version, _, err := tf.Version(ctx) - if err != nil { - return nil, err + pf, _ := mod.ParsedFiles() + if len(pf) > 0 { + merger.SetParsedFiles(pf) } - mm.logger.Printf("Terraform version %s found at %s (alternative)", version, tf.GetExecPath()) - return tf.Format, nil + return merger.MergeWithJsonProviderSchemas(providerSchema) } -func (mm *moduleManager) IsTerraformAvailable(path string) (bool, error) { - mod, err := mm.ModuleByPath(path) +func (mm *moduleManager) SchemaSourcesForModule(modPath string) ([]SchemaSource, error) { + mod, err := mm.ModuleByPath(modPath) if err != nil { - return false, err + return []SchemaSource{}, err } - return mod.IsTerraformAvailable(), nil -} - -func (mm *moduleManager) HasTerraformDiscoveryFinished(path string) (bool, error) { - mod, err := mm.ModuleByPath(path) - if err != nil { - return false, err + if ps, err := mod.ProviderSchema(); err == nil && ps != nil { + return []SchemaSource{mod}, nil } - return mod.HasTerraformDiscoveryFinished(), nil -} - -func (mm *moduleManager) CancelLoading() { + sources := make([]SchemaSource, 0) for _, mod := range mm.modules { - mm.logger.Printf("cancelling loading for %s", mod.Path()) - mod.CancelLoading() - mm.logger.Printf("loading cancelled for %s", mod.Path()) - } - mm.workerPool.Stop() -} - -// trimLockFilePath strips known lock file paths and filenames -// to get the directory path of the relevant module -func trimLockFilePath(filePath string) string { - pluginLockFileSuffixes := pluginLockFilePaths(string(os.PathSeparator)) - for _, s := range pluginLockFileSuffixes { - if strings.HasSuffix(filePath, s) { - return strings.TrimSuffix(filePath, s) + if mod.CallsModule(modPath) { + if ps, err := mod.ProviderSchema(); err == nil && ps != nil { + sources = append(sources, mod) + } } } - moduleManifestSuffix := moduleManifestFilePath(string(os.PathSeparator)) - if strings.HasSuffix(filePath, moduleManifestSuffix) { - return strings.TrimSuffix(filePath, moduleManifestSuffix) - } + // We could expose preloaded schemas here already + // but other logic elsewhere isn't able to take advantage + // of multiple sources and mix-and-match yet. + // TODO https://github.com/hashicorp/terraform-ls/issues/354 - return filePath + return sources, nil } -func (mm *moduleManager) PathsToWatch() []string { - paths := make([]string, 0) +func (mm *moduleManager) moduleByPath(dir string) (*module, bool) { for _, mod := range mm.modules { - ptw := mod.PathsToWatch() - if len(ptw) > 0 { - paths = append(paths, ptw...) + if pathEquals(mod.Path(), dir) { + return mod, true } } - return paths + return nil, false } -// NewModuleLoader allows adding & loading modules -// with a given context. This can be passed down to any handler -// which itself will have short-lived context -// therefore couldn't finish loading the module asynchronously -// after it responds to the client -func NewModuleLoader(ctx context.Context, mm ModuleManager) ModuleLoader { - return func(dir string) (Module, error) { - return mm.AddAndStartLoadingModule(ctx, dir) +func (mm *moduleManager) ListModules() []Module { + modules := make([]Module, 0) + for _, mod := range mm.modules { + modules = append(modules, mod) } + return modules +} +func (mm *moduleManager) ModuleByPath(path string) (Module, error) { + path = filepath.Clean(path) + + if mod, ok := mm.moduleByPath(path); ok { + return mod, nil + } + + return nil, &ModuleNotFoundErr{path} +} + +func (mm *moduleManager) CancelLoading() { + mm.cancelFunc() } diff --git a/internal/terraform/module/module_manager_mock.go b/internal/terraform/module/module_manager_mock.go index ceeff6ec7..cbcbdba6c 100644 --- a/internal/terraform/module/module_manager_mock.go +++ b/internal/terraform/module/module_manager_mock.go @@ -2,62 +2,78 @@ package module import ( "context" - "fmt" "log" + "github.com/hashicorp/go-version" + "github.com/hashicorp/terraform-json" "github.com/hashicorp/terraform-ls/internal/filesystem" - "github.com/hashicorp/terraform-ls/internal/terraform/discovery" "github.com/hashicorp/terraform-ls/internal/terraform/exec" + "github.com/stretchr/testify/mock" ) -type ModuleMockFactory struct { - mmocks map[string]*ModuleMock - logger *log.Logger - fs filesystem.Filesystem -} - -func (mmf *ModuleMockFactory) New(ctx context.Context, dir string) (*module, error) { - mmocks, ok := mmf.mmocks[dir] - if !ok { - return nil, fmt.Errorf("unexpected module requested: %s (%d available: %#v)", dir, len(mmf.mmocks), mmf.mmocks) - } - - mock := NewModuleMock(mmocks, mmf.fs, dir) - mock.SetLogger(mmf.logger) - return mock, mock.discoverCaches(ctx, dir) -} - type ModuleManagerMockInput struct { - Modules map[string]*ModuleMock - TfExecutorFactory exec.ExecutorFactory + Logger *log.Logger + TerraformCalls *exec.TerraformMockCalls } func NewModuleManagerMock(input *ModuleManagerMockInput) ModuleManagerFactory { - return func(fs filesystem.Filesystem) ModuleManager { - mm := newModuleManager(fs) - mm.syncLoading = true - - mmf := &ModuleMockFactory{ - mmocks: make(map[string]*ModuleMock, 0), - logger: mm.logger, - fs: fs, - } + var logger *log.Logger + var tfCalls *exec.TerraformMockCalls - // mock terraform discovery - md := &discovery.MockDiscovery{Path: "tf-mock"} - mm.tfDiscoFunc = md.LookPath - - // mock terraform executor - if input != nil { - mm.tfNewExecutor = input.TfExecutorFactory + if input != nil { + logger = input.Logger + tfCalls = input.TerraformCalls + } - if input.Modules != nil { - mmf.mmocks = input.Modules - } + return func(ctx context.Context, fs filesystem.Filesystem) ModuleManager { + if tfCalls != nil { + ctx = exec.WithExecutorFactory(ctx, exec.NewMockExecutor(tfCalls)) + ctx = exec.WithExecutorOpts(ctx, &exec.ExecutorOpts{ + ExecPath: "tf-mock", + }) } - mm.newModule = mmf.New + mm := NewSyncModuleManager(ctx, fs) + + if logger != nil { + mm.SetLogger(logger) + } return mm } } + +func validTfMockCalls(repeatability int) []*mock.Call { + return []*mock.Call{ + { + Method: "Version", + Repeatability: repeatability, + Arguments: []interface{}{ + mock.AnythingOfType("*context.cancelCtx"), + }, + ReturnArguments: []interface{}{ + version.Must(version.NewVersion("0.12.0")), + nil, + nil, + }, + }, + { + Method: "GetExecPath", + Repeatability: repeatability, + ReturnArguments: []interface{}{ + "", + }, + }, + { + Method: "ProviderSchemas", + Repeatability: repeatability, + Arguments: []interface{}{ + mock.AnythingOfType("*context.cancelCtx"), + }, + ReturnArguments: []interface{}{ + &tfjson.ProviderSchemas{FormatVersion: "0.1"}, + nil, + }, + }, + } +} diff --git a/internal/terraform/module/module_manager_mock_test.go b/internal/terraform/module/module_manager_mock_test.go deleted file mode 100644 index 8dbbdc712..000000000 --- a/internal/terraform/module/module_manager_mock_test.go +++ /dev/null @@ -1,74 +0,0 @@ -package module - -import ( - "context" - "os" - "path/filepath" - "testing" - - "github.com/hashicorp/go-version" - tfjson "github.com/hashicorp/terraform-json" - "github.com/hashicorp/terraform-ls/internal/filesystem" - "github.com/hashicorp/terraform-ls/internal/terraform/exec" - "github.com/stretchr/testify/mock" -) - -func TestNewModuleManagerMock_noMocks(t *testing.T) { - f := NewModuleManagerMock(nil) - mm := f(filesystem.NewFilesystem()) - _, err := mm.AddAndStartLoadingModule(context.Background(), "any-path") - if err == nil { - t.Fatal("expected unmocked path addition to fail") - } -} - -func TestNewModuleManagerMock_mocks(t *testing.T) { - tmpDir := filepath.Clean(os.TempDir()) - - f := NewModuleManagerMock(&ModuleManagerMockInput{ - Modules: map[string]*ModuleMock{ - tmpDir: { - TfExecFactory: validTfMockCalls(t, tmpDir), - }, - }}) - mm := f(filesystem.NewFilesystem()) - _, err := mm.AddAndStartLoadingModule(context.Background(), tmpDir) - if err != nil { - t.Fatal(err) - } -} - -func validTfMockCalls(t *testing.T, workDir string) exec.ExecutorFactory { - return exec.NewMockExecutor([]*mock.Call{ - { - Method: "Version", - Repeatability: 1, - Arguments: []interface{}{ - mock.AnythingOfType("*context.emptyCtx"), - }, - ReturnArguments: []interface{}{ - version.Must(version.NewVersion("0.12.0")), - nil, - nil, - }, - }, - { - Method: "GetExecPath", - Repeatability: 1, - ReturnArguments: []interface{}{ - "", - }, - }, - { - Method: "ProviderSchemas", - Repeatability: 1, - Arguments: []interface{}{ - mock.AnythingOfType("*context.emptyCtx"), - }, - ReturnArguments: []interface{}{ - &tfjson.ProviderSchemas{FormatVersion: "0.1"}, - nil, - }, - }, - }) -} diff --git a/internal/terraform/module/module_manager_test.go b/internal/terraform/module/module_manager_test.go index a8ee59097..eb3fcb13c 100644 --- a/internal/terraform/module/module_manager_test.go +++ b/internal/terraform/module/module_manager_test.go @@ -10,12 +10,8 @@ import ( "testing" "github.com/google/go-cmp/cmp" - "github.com/hashicorp/go-version" - tfjson "github.com/hashicorp/terraform-json" "github.com/hashicorp/terraform-ls/internal/filesystem" - "github.com/hashicorp/terraform-ls/internal/terraform/discovery" "github.com/hashicorp/terraform-ls/internal/terraform/exec" - "github.com/stretchr/testify/mock" ) func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { @@ -27,58 +23,33 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { testCases := []struct { name string walkerRoot string + totalModuleCount int lookupPath string expectedCandidates []string }{ - { - // outside of watcher, modules are always looked up by dir - "tf-file-based lookup", - filepath.Join(testData, "single-root-ext-modules-only"), - filepath.Join(testData, "single-root-ext-modules-only", "main.tf"), - []string{}, - }, { "dir-based lookup (exact match)", filepath.Join(testData, "single-root-ext-modules-only"), + 1, filepath.Join(testData, "single-root-ext-modules-only"), []string{ filepath.Join(testData, "single-root-ext-modules-only"), }, }, - { - "lock-file-based lookup", - filepath.Join(testData, "single-root-ext-modules-only"), - filepath.Join(testData, "single-root-ext-modules-only", - ".terraform", - "modules", - "modules.json"), - []string{ - filepath.Join(testData, "single-root-ext-modules-only"), - }, - }, { "dir-based lookup (exact match)", filepath.Join(testData, "single-root-local-and-ext-modules"), + 1, filepath.Join(testData, "single-root-local-and-ext-modules"), []string{ filepath.Join(testData, "single-root-local-and-ext-modules"), }, }, - { - "lock-file-based lookup", - filepath.Join(testData, "single-root-local-and-ext-modules"), - filepath.Join(testData, "single-root-local-and-ext-modules", - ".terraform", - "modules", - "modules.json"), - []string{ - filepath.Join(testData, "single-root-local-and-ext-modules"), - }, - }, { "mod-ref-based lookup", filepath.Join(testData, "single-root-local-and-ext-modules"), + 1, filepath.Join(testData, "single-root-local-and-ext-modules/alpha"), []string{ filepath.Join(testData, "single-root-local-and-ext-modules"), @@ -87,6 +58,7 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "mod-ref-based lookup", filepath.Join(testData, "single-root-local-and-ext-modules"), + 1, filepath.Join(testData, "single-root-local-and-ext-modules/beta"), []string{ filepath.Join(testData, "single-root-local-and-ext-modules"), @@ -95,6 +67,7 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "mod-ref-based lookup (not referenced)", filepath.Join(testData, "single-root-local-and-ext-modules"), + 1, filepath.Join(testData, "single-root-local-and-ext-modules/charlie"), []string{}, }, @@ -102,25 +75,16 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "dir-based lookup (exact match)", filepath.Join(testData, "single-root-local-modules-only"), + 1, filepath.Join(testData, "single-root-local-modules-only"), []string{ filepath.Join(testData, "single-root-local-modules-only"), }, }, - { - "lock-file-based lookup", - filepath.Join(testData, "single-root-local-modules-only"), - filepath.Join(testData, "single-root-local-modules-only", - ".terraform", - "modules", - "modules.json"), - []string{ - filepath.Join(testData, "single-root-local-modules-only"), - }, - }, { "mod-ref-based lookup", filepath.Join(testData, "single-root-local-modules-only"), + 1, filepath.Join(testData, "single-root-local-modules-only/alpha"), []string{ filepath.Join(testData, "single-root-local-modules-only"), @@ -129,6 +93,7 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "mod-ref-based lookup", filepath.Join(testData, "single-root-local-modules-only"), + 1, filepath.Join(testData, "single-root-local-modules-only/beta"), []string{ filepath.Join(testData, "single-root-local-modules-only"), @@ -137,6 +102,7 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "mod-ref-based lookup (not referenced)", filepath.Join(testData, "single-root-local-modules-only"), + 1, filepath.Join(testData, "single-root-local-modules-only/charlie"), []string{}, }, @@ -144,85 +110,46 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "dir-based lookup (exact match)", filepath.Join(testData, "single-root-no-modules"), + 1, filepath.Join(testData, "single-root-no-modules"), []string{ filepath.Join(testData, "single-root-no-modules"), }, }, - { - "lock-file-based lookup", - filepath.Join(testData, "single-root-no-modules"), - filepath.Join(testData, "single-root-no-modules", - ".terraform", - "modules", - "modules.json"), - []string{ - filepath.Join(testData, "single-root-no-modules"), - }, - }, { "directory-based lookup", filepath.Join(testData, "nested-single-root-no-modules"), + 1, filepath.Join(testData, "nested-single-root-no-modules", "tf-root"), []string{ filepath.Join(testData, "nested-single-root-no-modules", "tf-root"), }, }, - { - "lock-file-based lookup", - filepath.Join(testData, "nested-single-root-no-modules"), - filepath.Join(testData, "nested-single-root-no-modules", "tf-root", - ".terraform", - "modules", - "modules.json"), - []string{ - filepath.Join(testData, "nested-single-root-no-modules", "tf-root"), - }, - }, { "directory-based lookup", filepath.Join(testData, "nested-single-root-ext-modules-only"), + 1, filepath.Join(testData, "nested-single-root-ext-modules-only", "tf-root"), []string{ filepath.Join(testData, "nested-single-root-ext-modules-only", "tf-root"), }, }, - { - "lock-file-based lookup", - filepath.Join(testData, "nested-single-root-ext-modules-only"), - filepath.Join(testData, "nested-single-root-ext-modules-only", "tf-root", - ".terraform", - "modules", - "modules.json"), - []string{ - filepath.Join(testData, "nested-single-root-ext-modules-only", "tf-root"), - }, - }, { "directory-based lookup", filepath.Join(testData, "nested-single-root-local-modules-down"), + 1, filepath.Join(testData, "nested-single-root-local-modules-down", "tf-root"), []string{ filepath.Join(testData, "nested-single-root-local-modules-down", "tf-root"), }, }, - { - "lock-file-based lookup", - filepath.Join(testData, "nested-single-root-local-modules-down"), - filepath.Join(testData, "nested-single-root-local-modules-down", "tf-root", - ".terraform", - "modules", - "modules.json"), - []string{ - filepath.Join(testData, "nested-single-root-local-modules-down", "tf-root"), - }, - }, { "mod-based lookup", filepath.Join(testData, "nested-single-root-local-modules-down"), + 1, filepath.Join(testData, "nested-single-root-local-modules-down", "tf-root", "alpha"), []string{ filepath.Join(testData, "nested-single-root-local-modules-down", "tf-root"), @@ -231,12 +158,14 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "mod-based lookup", filepath.Join(testData, "nested-single-root-local-modules-down"), + 1, filepath.Join(testData, "nested-single-root-local-modules-down", "tf-root", "beta"), []string{}, }, { "mod-based lookup", filepath.Join(testData, "nested-single-root-local-modules-down"), + 1, filepath.Join(testData, "nested-single-root-local-modules-down", "tf-root", "charlie"), []string{ filepath.Join(testData, "nested-single-root-local-modules-down", "tf-root"), @@ -246,25 +175,16 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "dir-based lookup", filepath.Join(testData, "nested-single-root-local-modules-up"), + 1, filepath.Join(testData, "nested-single-root-local-modules-up", "module", "tf-root"), []string{ filepath.Join(testData, "nested-single-root-local-modules-up", "module", "tf-root"), }, }, - { - "lock-file-based lookup", - filepath.Join(testData, "nested-single-root-local-modules-up"), - filepath.Join(testData, "nested-single-root-local-modules-up", "module", "tf-root", - ".terraform", - "modules", - "modules.json"), - []string{ - filepath.Join(testData, "nested-single-root-local-modules-up", "module", "tf-root"), - }, - }, { "mod-based lookup", filepath.Join(testData, "nested-single-root-local-modules-up"), + 1, filepath.Join(testData, "nested-single-root-local-modules-up", "module"), []string{ filepath.Join(testData, "nested-single-root-local-modules-up", "module", "tf-root"), @@ -276,6 +196,7 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "directory-env-based lookup", filepath.Join(testData, "main-module-multienv"), + 3, filepath.Join(testData, "main-module-multienv", "env", "dev"), []string{ filepath.Join(testData, "main-module-multienv", "env", "dev"), @@ -284,6 +205,7 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "directory-env-based lookup", filepath.Join(testData, "main-module-multienv"), + 3, filepath.Join(testData, "main-module-multienv", "env", "prod"), []string{ filepath.Join(testData, "main-module-multienv", "env", "prod"), @@ -292,6 +214,7 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "main module lookup", filepath.Join(testData, "main-module-multienv"), + 3, filepath.Join(testData, "main-module-multienv", "main"), []string{ filepath.Join(testData, "main-module-multienv", "env", "dev"), @@ -303,6 +226,7 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "dir-based lookup", filepath.Join(testData, "multi-root-no-modules"), + 3, filepath.Join(testData, "multi-root-no-modules", "first-root"), []string{ filepath.Join(testData, "multi-root-no-modules", "first-root"), @@ -311,56 +235,26 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "dir-based lookup", filepath.Join(testData, "multi-root-no-modules"), + 3, filepath.Join(testData, "multi-root-no-modules", "second-root"), []string{ filepath.Join(testData, "multi-root-no-modules", "second-root"), }, }, - { - "lock-file-based lookup", - filepath.Join(testData, "multi-root-no-modules"), - filepath.Join(testData, "multi-root-no-modules", "first-root", - ".terraform", - "modules", - "modules.json"), - []string{ - filepath.Join(testData, "multi-root-no-modules", "first-root"), - }, - }, - { - "lock-file-based lookup", - filepath.Join(testData, "multi-root-no-modules"), - filepath.Join(testData, "multi-root-no-modules", "second-root", - ".terraform", - "modules", - "modules.json"), - []string{ - filepath.Join(testData, "multi-root-no-modules", "second-root"), - }, - }, { "dir-based lookup", filepath.Join(testData, "multi-root-local-modules-down"), + 3, filepath.Join(testData, "multi-root-local-modules-down", "first-root"), []string{ filepath.Join(testData, "multi-root-local-modules-down", "first-root"), }, }, - { - "lock-file-based lookup", - filepath.Join(testData, "multi-root-local-modules-down"), - filepath.Join(testData, "multi-root-local-modules-down", "first-root", - ".terraform", - "modules", - "modules.json"), - []string{ - filepath.Join(testData, "multi-root-local-modules-down", "first-root"), - }, - }, { "mod-based lookup", filepath.Join(testData, "multi-root-local-modules-down"), + 3, filepath.Join(testData, "multi-root-local-modules-down", "first-root", "alpha"), []string{ filepath.Join(testData, "multi-root-local-modules-down", "first-root"), @@ -369,12 +263,14 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "mod-based lookup", filepath.Join(testData, "multi-root-local-modules-down"), + 3, filepath.Join(testData, "multi-root-local-modules-down", "first-root", "beta"), []string{}, }, { "mod-based lookup", filepath.Join(testData, "multi-root-local-modules-down"), + 3, filepath.Join(testData, "multi-root-local-modules-down", "first-root", "charlie"), []string{ filepath.Join(testData, "multi-root-local-modules-down", "first-root"), @@ -383,25 +279,16 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "dir-based lookup", filepath.Join(testData, "multi-root-local-modules-down"), + 3, filepath.Join(testData, "multi-root-local-modules-down", "second-root"), []string{ filepath.Join(testData, "multi-root-local-modules-down", "second-root"), }, }, - { - "lock-file-based lookup", - filepath.Join(testData, "multi-root-local-modules-down"), - filepath.Join(testData, "multi-root-local-modules-down", "second-root", - ".terraform", - "modules", - "modules.json"), - []string{ - filepath.Join(testData, "multi-root-local-modules-down", "second-root"), - }, - }, { "mod-based lookup", filepath.Join(testData, "multi-root-local-modules-down"), + 3, filepath.Join(testData, "multi-root-local-modules-down", "second-root", "alpha"), []string{ filepath.Join(testData, "multi-root-local-modules-down", "second-root"), @@ -410,12 +297,14 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "mod-based lookup", filepath.Join(testData, "multi-root-local-modules-down"), + 3, filepath.Join(testData, "multi-root-local-modules-down", "second-root", "beta"), []string{}, }, { "mod-based lookup", filepath.Join(testData, "multi-root-local-modules-down"), + 3, filepath.Join(testData, "multi-root-local-modules-down", "second-root", "charlie"), []string{ filepath.Join(testData, "multi-root-local-modules-down", "second-root"), @@ -425,6 +314,7 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { { "dir-based lookup", filepath.Join(testData, "multi-root-local-modules-up"), + 3, filepath.Join(testData, "multi-root-local-modules-up", "main-module"), []string{ filepath.Join(testData, "multi-root-local-modules-up", "main-module", "modules", "first"), @@ -437,28 +327,57 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { for i, tc := range testCases { base := filepath.Base(tc.walkerRoot) t.Run(fmt.Sprintf("%d-%s/%s", i, tc.name, base), func(t *testing.T) { - mm := testModuleManager(t) - w := MockWalker() - w.SetLogger(testLogger()) ctx := context.Background() - err := w.StartWalking(ctx, tc.walkerRoot, func(ctx context.Context, modPath string) error { - _, err := mm.AddAndStartLoadingModule(ctx, modPath) - return err + fs := filesystem.NewFilesystem() + mmock := NewModuleManagerMock(&ModuleManagerMockInput{ + Logger: testLogger(), + TerraformCalls: &exec.TerraformMockCalls{ + AnyWorkDir: validTfMockCalls(tc.totalModuleCount), + }, }) + mm := mmock(ctx, fs) + t.Cleanup(mm.CancelLoading) + + w := SyncWalker(fs, mm) + w.SetLogger(testLogger()) + err := w.StartWalking(ctx, tc.walkerRoot) if err != nil { t.Fatal(err) } - candidates := mm.ModuleCandidatesByPath(tc.lookupPath) - if diff := cmp.Diff(tc.expectedCandidates, candidates.Paths()); diff != "" { + mm.AddModule(tc.lookupPath) + + candidates, err := mm.SchemaSourcesForModule(tc.lookupPath) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(tc.expectedCandidates, schemaSourcesPaths(t, candidates)); diff != "" { t.Fatalf("candidates don't match: %s", diff) } }) } } -func TestSchemaForPath_uninitialized(t *testing.T) { - mm := testModuleManager(t) +func schemaSourcesPaths(t *testing.T, srcs []SchemaSource) []string { + paths := make([]string, len(srcs)) + for i, src := range srcs { + mod, ok := src.(Module) + if !ok { + t.Fatal("schema source is not Module compatible") + } + paths[i] = mod.Path() + } + + return paths +} + +func TestSchemaForModule_uninitialized(t *testing.T) { + mmock := NewModuleManagerMock(nil) + + ctx := context.Background() + fs := filesystem.NewFilesystem() + mm := mmock(ctx, fs) + t.Cleanup(mm.CancelLoading) testData, err := filepath.Abs("testdata") if err != nil { @@ -466,80 +385,17 @@ func TestSchemaForPath_uninitialized(t *testing.T) { } path := filepath.Join(testData, "uninitialized-root") - // model is added automatically during didOpen - _, err = mm.AddAndStartLoadingModule(context.Background(), path) + _, err = mm.AddModule(path) if err != nil { t.Fatal(err) } - _, err = mm.SchemaForPath(path) + _, err = mm.SchemaForModule(path) if err != nil { t.Fatal(err) } } -func testModuleManager(t *testing.T) *moduleManager { - fs := filesystem.NewFilesystem() - mm := newModuleManager(fs) - mm.syncLoading = true - mm.logger = testLogger() - - mm.newModule = func(ctx context.Context, dir string) (*module, error) { - // TODO(RS): Should be just 1, unsure why it requires 2 - repeatability := 2 - mod := NewModuleMock(&ModuleMock{ - TfExecFactory: exec.NewMockExecutor([]*mock.Call{ - { - Method: "Version", - Repeatability: repeatability, - Arguments: []interface{}{ - mock.AnythingOfType(""), - }, - ReturnArguments: []interface{}{ - version.Must(version.NewVersion("0.12.0")), - nil, - nil, - }, - }, - { - Method: "GetExecPath", - Repeatability: repeatability, - ReturnArguments: []interface{}{ - "", - }, - }, - { - Method: "ProviderSchemas", - Repeatability: repeatability, - Arguments: []interface{}{ - mock.AnythingOfType(""), - }, - ReturnArguments: []interface{}{ - &tfjson.ProviderSchemas{FormatVersion: "0.1"}, - nil, - }, - }, - }), - }, fs, dir) - mod.logger = testLogger() - md := &discovery.MockDiscovery{Path: "tf-mock"} - mod.tfDiscoFunc = md.LookPath - - err := mod.discoverCaches(ctx, dir) - if err != nil { - t.Fatal(err) - } - - err = mod.load(ctx) - if err != nil { - t.Fatal(err) - } - - return mod, nil - } - return mm -} - func testLogger() *log.Logger { if testing.Verbose() { return log.New(os.Stdout, "", log.LstdFlags|log.Lshortfile) diff --git a/internal/terraform/module/module_manifest_test.go b/internal/terraform/module/module_manifest_test.go deleted file mode 100644 index 84a6c51ec..000000000 --- a/internal/terraform/module/module_manifest_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package module - -const moduleManifestRecord_external = `{ - "Key": "web_server_sg", - "Source": "terraform-aws-modules/security-group/aws//modules/http-80", - "Version": "3.10.0", - "Dir": ".terraform/modules/web_server_sg/terraform-aws-security-group-3.10.0/modules/http-80" -}` - -const moduleManifestRecord_externalDirtyPath = `{ - "Key": "web_server_sg", - "Source": "terraform-aws-modules/security-group/aws//modules/http-80", - "Version": "3.10.0", - "Dir": ".terraform/modules/web_server_sg/terraform-aws-security-group-3.10.0/modules/something/../http-80" -}` - -const moduleManifestRecord_local = `{ - "Key": "local", - "Source": "./nested/path", - "Dir": "nested/path" -}` - -const moduleManifestRecord_root = `{ - "Key": "", - "Source": "", - "Dir": "." -}` diff --git a/internal/terraform/module/module_mock.go b/internal/terraform/module/module_mock.go deleted file mode 100644 index 0c24a7e16..000000000 --- a/internal/terraform/module/module_mock.go +++ /dev/null @@ -1,30 +0,0 @@ -package module - -import ( - tfjson "github.com/hashicorp/terraform-json" - "github.com/hashicorp/terraform-ls/internal/filesystem" - "github.com/hashicorp/terraform-ls/internal/terraform/discovery" - "github.com/hashicorp/terraform-ls/internal/terraform/exec" -) - -type ModuleMock struct { - TfExecFactory exec.ExecutorFactory - ProviderSchemas *tfjson.ProviderSchemas -} - -func NewModuleMock(modMock *ModuleMock, fs filesystem.Filesystem, dir string) *module { - module := newModule(fs, dir) - - // mock terraform discovery - md := &discovery.MockDiscovery{Path: "tf-mock"} - module.tfDiscoFunc = md.LookPath - - // mock terraform executor - module.tfNewExecutor = modMock.TfExecFactory - - if modMock.ProviderSchemas != nil { - module.providerSchema = modMock.ProviderSchemas - } - - return module -} diff --git a/internal/terraform/module/module_ops.go b/internal/terraform/module/module_ops.go new file mode 100644 index 000000000..114a9b70d --- /dev/null +++ b/internal/terraform/module/module_ops.go @@ -0,0 +1,214 @@ +package module + +import ( + "context" + "fmt" + "path/filepath" + "strings" + + "github.com/hashicorp/hcl-lang/decoder" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/hashicorp/terraform-ls/internal/terraform/datadir" +) + +type OpState uint + +const ( + OpStateUnknown OpState = iota + OpStateQueued + OpStateLoading + OpStateLoaded +) + +type OpType uint + +const ( + OpTypeUnknown OpType = iota + OpTypeGetTerraformVersion + OpTypeObtainSchema + OpTypeParseConfiguration + OpTypeParseModuleManifest +) + +func (t OpType) String() string { + switch t { + case OpTypeUnknown: + return "OpTypeUnknown" + case OpTypeGetTerraformVersion: + return "OpTypeGetTerraformVersion" + case OpTypeObtainSchema: + return "OpTypeObtainSchema" + case OpTypeParseConfiguration: + return "OpTypeParseConfiguration" + case OpTypeParseModuleManifest: + return "OpTypeParseModuleManifest" + } + + return fmt.Sprintf("OpType(%d)", t) +} + +type ModuleOperation struct { + Module Module + Type OpType + + doneCh chan struct{} +} + +func NewModuleOperation(mod Module, typ OpType) ModuleOperation { + return ModuleOperation{ + Module: mod, + Type: typ, + doneCh: make(chan struct{}, 1), + } +} + +func (mo ModuleOperation) markAsDone() { + mo.doneCh <- struct{}{} +} + +func (mo ModuleOperation) Done() <-chan struct{} { + return mo.doneCh +} + +func GetTerraformVersion(ctx context.Context, mod Module) { + m := mod.(*module) + + m.SetTerraformVersionState(OpStateLoading) + defer m.SetTerraformVersionState(OpStateLoaded) + + tfExec, err := TerraformExecutorForModule(ctx, mod) + if err != nil { + m.SetTerraformVersion(nil, err) + m.logger.Printf("getting executor failed: %s", err) + return + } + + v, pv, err := tfExec.Version(ctx) + if err != nil { + m.logger.Printf("failed to get terraform version: %s", err) + } else { + m.logger.Printf("got terraform version successfully for %s", m.Path()) + } + + m.SetTerraformVersion(v, err) + if len(pv) > 0 { + m.SetProviderVersions(pv) + } +} + +func ObtainSchema(ctx context.Context, mod Module) { + m := mod.(*module) + m.SetProviderSchemaObtainingState(OpStateLoading) + defer m.SetProviderSchemaObtainingState(OpStateLoaded) + + tfExec, err := TerraformExecutorForModule(ctx, mod) + if err != nil { + m.SetProviderSchemas(nil, err) + m.logger.Printf("getting executor failed: %s", err) + return + } + + ps, err := tfExec.ProviderSchemas(ctx) + if err != nil { + m.logger.Printf("failed to obtain schema: %s", err) + } else { + m.logger.Printf("schema obtained successfully for %s", m.Path()) + } + + m.SetProviderSchemas(ps, err) +} + +func ParseConfiguration(mod Module) { + m := mod.(*module) + m.SetConfigParsingState(OpStateLoading) + defer m.SetConfigParsingState(OpStateLoaded) + + files := make(map[string]*hcl.File, 0) + diags := make(map[string]hcl.Diagnostics, 0) + + infos, err := m.fs.ReadDir(m.Path()) + if err != nil { + m.SetParsedFiles(files, err) + return + } + + for _, info := range infos { + if info.IsDir() { + // We only care about files + continue + } + + name := info.Name() + if !strings.HasSuffix(name, ".tf") || IsIgnoredFile(name) { + continue + } + + // TODO: overrides + + fullPath := filepath.Join(m.Path(), name) + + src, err := m.fs.ReadFile(fullPath) + if err != nil { + m.SetParsedFiles(files, err) + return + } + + f, pDiags := hclsyntax.ParseConfig(src, name, hcl.InitialPos) + diags[name] = pDiags + if f != nil { + files[name] = f + } + } + + m.SetParsedFiles(files, err) + m.SetDiagnostics(diags) + return +} + +// IsIgnoredFile returns true if the given filename (which must not have a +// directory path ahead of it) should be ignored as e.g. an editor swap file. +func IsIgnoredFile(name string) bool { + return strings.HasPrefix(name, ".") || // Unix-like hidden files + strings.HasSuffix(name, "~") || // vim + strings.HasPrefix(name, "#") && strings.HasSuffix(name, "#") // emacs +} + +func ParseModuleManifest(mod Module) { + m := mod.(*module) + m.SetModuleManifestParsingState(OpStateLoading) + defer m.SetModuleManifestParsingState(OpStateLoaded) + + manifestPath, ok := datadir.ModuleManifestFilePath(m.fs, mod.Path()) + if !ok { + m.logger.Printf("%s: manifest file does not exist", mod.Path()) + return + } + + mm, err := datadir.ParseModuleManifestFromFile(manifestPath) + if err != nil { + m.logger.Printf("failed to parse manifest: %s", err) + } else { + m.logger.Printf("manifest parsed successfully for %s", m.Path()) + } + + m.SetModuleManifest(mm, err) +} + +func DecoderForModule(mod Module) (*decoder.Decoder, error) { + d := decoder.NewDecoder() + + pf, err := mod.ParsedFiles() + if err != nil { + return nil, err + } + + for name, f := range pf { + err := d.LoadFile(name, f) + if err != nil { + return nil, fmt.Errorf("failed to load a file: %w", err) + } + } + + return d, nil +} diff --git a/internal/terraform/module/module_ops_queue.go b/internal/terraform/module/module_ops_queue.go new file mode 100644 index 000000000..f4d0677ab --- /dev/null +++ b/internal/terraform/module/module_ops_queue.go @@ -0,0 +1,99 @@ +package module + +import ( + "container/heap" + "sync" +) + +type moduleOpsQueue struct { + q queue + mu *sync.Mutex +} + +func newModuleOpsQueue() moduleOpsQueue { + q := moduleOpsQueue{ + q: make(queue, 0), + mu: &sync.Mutex{}, + } + heap.Init(&q.q) + return q +} + +func (q *moduleOpsQueue) PushOp(op ModuleOperation) { + q.mu.Lock() + defer q.mu.Unlock() + + heap.Push(&q.q, op) + +} + +func (q *moduleOpsQueue) PopOp() ModuleOperation { + q.mu.Lock() + defer q.mu.Unlock() + + item := heap.Pop(&q.q) + modOp := item.(ModuleOperation) + return modOp +} + +func (q *moduleOpsQueue) Len() int { + q.mu.Lock() + defer q.mu.Unlock() + + return q.q.Len() +} + +func (q *moduleOpsQueue) Peek() interface{} { + q.mu.Lock() + defer q.mu.Unlock() + + item := q.q.Peek() + return item +} + +type queue []ModuleOperation + +var _ heap.Interface = &queue{} + +func (q *queue) Push(x interface{}) { + modOp := x.(ModuleOperation) + *q = append(*q, modOp) +} + +func (q queue) Swap(i, j int) { + q[i], q[j] = q[j], q[i] +} + +func (q *queue) Pop() interface{} { + old := *q + n := len(old) + item := old[n-1] + *q = old[0 : n-1] + return item +} + +func (q queue) Peek() interface{} { + n := len(q) + return q[n-1] +} + +func (q queue) Len() int { + return len(q) +} + +func (q queue) Less(i, j int) bool { + return moduleOperationLess(q[i], q[j]) +} + +func moduleOperationLess(aModOp, bModOp ModuleOperation) bool { + leftOpen, rightOpen := 0, 0 + + if aModOp.Module.HasOpenFiles() { + leftOpen = 1 + } + if bModOp.Module.HasOpenFiles() { + rightOpen = 1 + } + + return leftOpen > rightOpen +} diff --git a/internal/terraform/module/module_ops_queue_test.go b/internal/terraform/module/module_ops_queue_test.go new file mode 100644 index 000000000..d43c06b7d --- /dev/null +++ b/internal/terraform/module/module_ops_queue_test.go @@ -0,0 +1,91 @@ +package module + +import ( + "os" + "path/filepath" + "testing" + + "github.com/hashicorp/terraform-ls/internal/filesystem" + ilsp "github.com/hashicorp/terraform-ls/internal/lsp" + "github.com/hashicorp/terraform-ls/internal/protocol" + "github.com/hashicorp/terraform-ls/internal/uri" +) + +func TestModuleOpsQueue_modulePriority(t *testing.T) { + mq := newModuleOpsQueue() + + fs := filesystem.NewFilesystem() + fs.SetLogger(testLogger()) + + dir := t.TempDir() + t.Cleanup(func() { + os.RemoveAll(dir) + }) + + ops := []ModuleOperation{ + NewModuleOperation( + closedModAtPath(t, fs, dir, "alpha"), + OpTypeGetTerraformVersion, + ), + NewModuleOperation( + openModAtPath(t, fs, dir, "beta"), + OpTypeGetTerraformVersion, + ), + NewModuleOperation( + openModAtPath(t, fs, dir, "gamma"), + OpTypeGetTerraformVersion, + ), + NewModuleOperation( + closedModAtPath(t, fs, dir, "delta"), + OpTypeGetTerraformVersion, + ), + } + + for _, op := range ops { + mq.PushOp(op) + } + + firstOp := mq.PopOp() + + expectedFirstPath := filepath.Join(dir, "beta") + firstPath := firstOp.Module.Path() + if firstPath != expectedFirstPath { + t.Fatalf("path mismatch\nexpected: %s\ngiven: %s", + expectedFirstPath, firstPath) + } + + secondOp := mq.PopOp() + expectedSecondPath := filepath.Join(dir, "gamma") + secondPath := secondOp.Module.Path() + if secondPath != expectedSecondPath { + t.Fatalf("path mismatch\nexpected: %s\ngiven: %s", + expectedSecondPath, secondPath) + } +} + +func closedModAtPath(t *testing.T, fs filesystem.Filesystem, dir, modName string) Module { + modPath := filepath.Join(dir, modName) + + docPath := filepath.Join(modPath, "main.tf") + dh := ilsp.FileHandlerFromDocumentURI(protocol.DocumentURI(uri.FromPath(docPath))) + err := fs.CreateDocument(dh, []byte{}) + if err != nil { + t.Fatal(err) + } + m := newModule(fs, modPath) + m.SetLogger(testLogger()) + return m +} + +func openModAtPath(t *testing.T, fs filesystem.Filesystem, dir, modName string) Module { + modPath := filepath.Join(dir, modName) + docPath := filepath.Join(modPath, "main.tf") + dh := ilsp.FileHandlerFromDocumentURI(protocol.DocumentURI(uri.FromPath(docPath))) + err := fs.CreateAndOpenDocument(dh, []byte{}) + if err != nil { + t.Fatal(err) + } + m := newModule(fs, modPath) + m.SetLogger(testLogger()) + return m +} diff --git a/internal/terraform/module/path.go b/internal/terraform/module/path.go index 035391a18..aa3f45b37 100644 --- a/internal/terraform/module/path.go +++ b/internal/terraform/module/path.go @@ -6,6 +6,8 @@ import ( ) func pathEquals(path1, path2 string) bool { + path1 = filepath.Clean(path1) + path2 = filepath.Clean(path2) volume1 := filepath.VolumeName(path1) volume2 := filepath.VolumeName(path2) return strings.EqualFold(volume1, volume2) && path1[len(volume1):] == path2[len(volume2):] diff --git a/internal/terraform/module/plugin_lock_file.go b/internal/terraform/module/plugin_lock_file.go deleted file mode 100644 index b160d5733..000000000 --- a/internal/terraform/module/plugin_lock_file.go +++ /dev/null @@ -1,26 +0,0 @@ -package module - -import ( - "path/filepath" - "runtime" -) - -func pluginLockFilePaths(dir string) []string { - return []string{ - // Terraform >= 0.14 - filepath.Join(dir, - ".terraform.lock.hcl", - ), - // Terraform >= v0.13 - filepath.Join(dir, - ".terraform", - "plugins", - "selections.json"), - // Terraform <= v0.12 - filepath.Join(dir, - ".terraform", - "plugins", - runtime.GOOS+"_"+runtime.GOARCH, - "lock.json"), - } -} diff --git a/internal/terraform/module/terraform_executor.go b/internal/terraform/module/terraform_executor.go new file mode 100644 index 000000000..87d4b9aaf --- /dev/null +++ b/internal/terraform/module/terraform_executor.go @@ -0,0 +1,42 @@ +package module + +import ( + "context" + "fmt" + + "github.com/hashicorp/terraform-ls/internal/terraform/exec" +) + +func TerraformExecutorForModule(ctx context.Context, mod Module) (exec.TerraformExecutor, error) { + newExecutor, ok := exec.ExecutorFactoryFromContext(ctx) + if !ok { + return nil, fmt.Errorf("no terraform executor provided") + } + + var tfExec exec.TerraformExecutor + var err error + + opts, ok := exec.ExecutorOptsFromContext(ctx) + if ok && opts.ExecPath != "" { + tfExec, err = newExecutor(mod.Path(), opts.ExecPath) + if err != nil { + return nil, err + } + } else if mod.TerraformExecPath() != "" { + tfExec, err = newExecutor(mod.Path(), mod.TerraformExecPath()) + if err != nil { + return nil, err + } + } else { + return nil, fmt.Errorf("no exec path provided for terraform") + } + + if ok && opts.ExecLogPath != "" { + tfExec.SetExecLogPath(opts.ExecLogPath) + } + if ok && opts.Timeout != 0 { + tfExec.SetTimeout(opts.Timeout) + } + + return tfExec, nil +} diff --git a/internal/terraform/module/types.go b/internal/terraform/module/types.go index 2e1ef5561..0df0b8380 100644 --- a/internal/terraform/module/types.go +++ b/internal/terraform/module/types.go @@ -3,93 +3,78 @@ package module import ( "context" "log" - "time" - "github.com/hashicorp/hcl-lang/decoder" + "github.com/hashicorp/go-version" "github.com/hashicorp/hcl-lang/schema" "github.com/hashicorp/hcl/v2" + tfjson "github.com/hashicorp/terraform-json" "github.com/hashicorp/terraform-ls/internal/filesystem" - "github.com/hashicorp/terraform-ls/internal/terraform/exec" + "github.com/hashicorp/terraform-ls/internal/terraform/datadir" ) type File interface { Path() string } -type TerraformFormatterFinder interface { - TerraformFormatterForDir(ctx context.Context, path string) (exec.Formatter, error) - HasTerraformDiscoveryFinished(path string) (bool, error) - IsTerraformAvailable(path string) (bool, error) +type SchemaSource interface { + // module specific methods + Path() string + HumanReadablePath(string) string + + ProviderSchema() (*tfjson.ProviderSchemas, error) + TerraformVersion() (*version.Version, error) + ProviderVersions() map[string]*version.Version } type ModuleFinder interface { - ModuleCandidatesByPath(path string) Modules ModuleByPath(path string) (Module, error) - SchemaForPath(path string) (*schema.BodySchema, error) + SchemaForModule(path string) (*schema.BodySchema, error) + SchemaSourcesForModule(path string) ([]SchemaSource, error) } type ModuleLoader func(dir string) (Module, error) type ModuleManager interface { ModuleFinder - TerraformFormatterFinder SetLogger(logger *log.Logger) - - SetTerraformExecPath(path string) - SetTerraformExecLogPath(logPath string) - SetTerraformExecTimeout(timeout time.Duration) - - InitAndUpdateModule(ctx context.Context, dir string) (Module, error) - AddAndStartLoadingModule(ctx context.Context, dir string) (Module, error) - WorkerPoolSize() int - WorkerQueueSize() int - ListModules() Modules - PathsToWatch() []string + AddModule(modPath string) (Module, error) + EnqueueModuleOp(modPath string, opType OpType) error + EnqueueModuleOpWait(modPath string, opType OpType) error + ListModules() []Module CancelLoading() } -type Modules []Module - -func (mods Modules) Paths() []string { - paths := make([]string, len(mods)) - for i, mod := range mods { - paths[i] = mod.Path() - } - return paths -} - type Module interface { Path() string - MatchesPath(path string) bool - LoadError() error - StartLoading() error - IsLoadingDone() bool - LoadingDone() <-chan struct{} - IsKnownPluginLockFile(path string) bool - IsKnownModuleManifestFile(path string) bool - PathsToWatch() []string - UpdateProviderSchemaCache(ctx context.Context, lockFile File) error - IsProviderSchemaLoaded() bool - UpdateModuleManifest(manifestFile File) error - Decoder() (*decoder.Decoder, error) - DecoderWithSchema(*schema.BodySchema) (*decoder.Decoder, error) - MergedSchema() (*schema.BodySchema, error) - IsParsed() bool - ParseFiles() error - ParsedDiagnostics() map[string]hcl.Diagnostics - TerraformFormatter() (exec.Formatter, error) - HasTerraformDiscoveryFinished() bool - IsTerraformAvailable() bool - ExecuteTerraformInit(ctx context.Context) error - ExecuteTerraformValidate(ctx context.Context) (map[string]hcl.Diagnostics, error) - Modules() []ModuleRecord HumanReadablePath(string) string - WasInitialized() (bool, error) + MatchesPath(path string) bool + HasOpenFiles() bool + + TerraformExecPath() string + TerraformVersion() (*version.Version, error) + ProviderVersions() map[string]*version.Version + ProviderSchema() (*tfjson.ProviderSchemas, error) + ModuleManifest() (*datadir.ModuleManifest, error) + + TerraformVersionState() OpState + ProviderSchemaState() OpState + + ParsedFiles() (map[string]*hcl.File, error) + Diagnostics() map[string]hcl.Diagnostics + ModuleCalls() []datadir.ModuleRecord } -type ModuleFactory func(context.Context, string) (*module, error) +type ModuleFactory func(string) (*module, error) + +type ModuleManagerFactory func(context.Context, filesystem.Filesystem) ModuleManager -type ModuleManagerFactory func(filesystem.Filesystem) ModuleManager +type WalkerFactory func(filesystem.Filesystem, ModuleManager) *Walker -type WalkerFactory func() *Walker +type Watcher interface { + Start() error + Stop() error + SetLogger(*log.Logger) + AddModule(string) error + IsModuleWatched(string) bool +} diff --git a/internal/terraform/module/walker.go b/internal/terraform/module/walker.go index 172549600..66a484017 100644 --- a/internal/terraform/module/walker.go +++ b/internal/terraform/module/walker.go @@ -8,6 +8,9 @@ import ( "os" "path/filepath" "sync" + + "github.com/hashicorp/terraform-ls/internal/filesystem" + "github.com/hashicorp/terraform-ls/internal/terraform/datadir" ) var ( @@ -24,8 +27,11 @@ var ( ) type Walker struct { - logger *log.Logger - sync bool + fs filesystem.Filesystem + modMgr ModuleManager + watcher Watcher + logger *log.Logger + sync bool walking bool walkingMu *sync.RWMutex @@ -35,8 +41,10 @@ type Walker struct { excludeModulePaths map[string]bool } -func NewWalker() *Walker { +func NewWalker(fs filesystem.Filesystem, modMgr ModuleManager) *Walker { return &Walker{ + fs: fs, + modMgr: modMgr, logger: discardLogger, walkingMu: &sync.RWMutex{}, doneCh: make(chan struct{}, 0), @@ -47,6 +55,10 @@ func (w *Walker) SetLogger(logger *log.Logger) { w.logger = logger } +func (w *Walker) SetWatcher(watcher Watcher) { + w.watcher = watcher +} + func (w *Walker) SetExcludeModulePaths(excludeModulePaths []string) { w.excludeModulePaths = make(map[string]bool) for _, path := range excludeModulePaths { @@ -54,8 +66,6 @@ func (w *Walker) SetExcludeModulePaths(excludeModulePaths []string) { } } -type WalkFunc func(ctx context.Context, rootModulePath string) error - func (w *Walker) Stop() { if w.cancelFunc != nil { w.cancelFunc() @@ -73,11 +83,7 @@ func (w *Walker) setWalking(isWalking bool) { w.walking = isWalking } -func (w *Walker) Done() <-chan struct{} { - return w.doneCh -} - -func (w *Walker) StartWalking(ctx context.Context, path string, wf WalkFunc) error { +func (w *Walker) StartWalking(ctx context.Context, path string) error { if w.IsWalking() { return fmt.Errorf("walker is already running") } @@ -89,18 +95,18 @@ func (w *Walker) StartWalking(ctx context.Context, path string, wf WalkFunc) err if w.sync { w.logger.Printf("synchronously walking through %s", path) - return w.walk(ctx, path, wf) + return w.walk(ctx, path) } - go func(w *Walker, path string, wf WalkFunc) { + go func(w *Walker, path string) { w.logger.Printf("asynchronously walking through %s", path) - err := w.walk(ctx, path, wf) + err := w.walk(ctx, path) if err != nil { w.logger.Printf("async walking through %s failed: %s", path, err) return } w.logger.Printf("async walking through %s finished", path) - }(w, path, wf) + }(w, path) return nil } @@ -112,9 +118,12 @@ func (w *Walker) IsWalking() bool { return w.walking } -func (w *Walker) walk(ctx context.Context, rootPath string, wf WalkFunc) error { +func (w *Walker) walk(ctx context.Context, rootPath string) error { defer w.Stop() + // We ignore the passed FS and instead read straight from OS FS + // because that would require reimplementing filepath.Walk and + // the data directory should never be on the virtual filesystem anyway err := filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error { select { case <-w.doneCh: @@ -137,9 +146,45 @@ func (w *Walker) walk(ctx context.Context, rootPath string, wf WalkFunc) error { return filepath.SkipDir } - if info.Name() == ".terraform" { + if info.Name() == datadir.DataDirName { w.logger.Printf("found module %s", dir) - return wf(ctx, dir) + + _, err := w.modMgr.ModuleByPath(dir) + if err != nil { + if IsModuleNotFound(err) { + _, err := w.modMgr.AddModule(dir) + if err != nil { + return err + } + } else { + return err + } + } + + err = w.modMgr.EnqueueModuleOp(dir, OpTypeGetTerraformVersion) + if err != nil { + return err + } + + dataDir := datadir.WalkDataDirOfModule(w.fs, dir) + if dataDir.ModuleManifestPath != "" { + err = w.modMgr.EnqueueModuleOp(dir, OpTypeParseModuleManifest) + if err != nil { + return err + } + } + if dataDir.PluginLockFilePath != "" { + err = w.modMgr.EnqueueModuleOp(dir, OpTypeObtainSchema) + if err != nil { + return err + } + } + + if w.watcher != nil { + w.watcher.AddModule(dir) + } + + return nil } if !info.IsDir() { diff --git a/internal/terraform/module/walker_mock.go b/internal/terraform/module/walker_mock.go index 1c4b55798..bebdb4536 100644 --- a/internal/terraform/module/walker_mock.go +++ b/internal/terraform/module/walker_mock.go @@ -1,7 +1,9 @@ package module -func MockWalker() *Walker { - w := NewWalker() +import "github.com/hashicorp/terraform-ls/internal/filesystem" + +func SyncWalker(fs filesystem.Filesystem, modMgr ModuleManager) *Walker { + w := NewWalker(fs, modMgr) w.sync = true return w } diff --git a/internal/terraform/module/watcher.go b/internal/terraform/module/watcher.go new file mode 100644 index 000000000..b41354c27 --- /dev/null +++ b/internal/terraform/module/watcher.go @@ -0,0 +1,234 @@ +package module + +import ( + "context" + "io/ioutil" + "log" + "os" + "path/filepath" + + "github.com/fsnotify/fsnotify" + "github.com/hashicorp/terraform-ls/internal/filesystem" + "github.com/hashicorp/terraform-ls/internal/terraform/datadir" +) + +// Watcher is a wrapper around native fsnotify.Watcher +// It provides the ability to detect actual file changes +// (rather than just events that may not be changing any bytes) +type watcher struct { + fw *fsnotify.Watcher + fs filesystem.Filesystem + modMgr ModuleManager + modules []*watchedModule + logger *log.Logger + + watching bool + cancelFunc context.CancelFunc +} + +type WatcherFactory func(filesystem.Filesystem, ModuleManager) (Watcher, error) + +type watchedModule struct { + Path string + Watched []string + Watchable *datadir.WatchablePaths +} + +func NewWatcher(fs filesystem.Filesystem, modMgr ModuleManager) (Watcher, error) { + fw, err := fsnotify.NewWatcher() + if err != nil { + return nil, err + } + + return &watcher{ + fw: fw, + fs: fs, + modMgr: modMgr, + logger: defaultLogger, + modules: make([]*watchedModule, 0), + }, nil +} + +var defaultLogger = log.New(ioutil.Discard, "", 0) + +func (w *watcher) SetLogger(logger *log.Logger) { + w.logger = logger +} + +func (w *watcher) IsModuleWatched(modPath string) bool { + modPath = filepath.Clean(modPath) + + for _, m := range w.modules { + if pathEquals(m.Path, modPath) { + return true + } + } + + return false +} + +func (w *watcher) AddModule(modPath string) error { + modPath = filepath.Clean(modPath) + + w.logger.Printf("adding module for watching: %s", modPath) + + wm := &watchedModule{ + Path: modPath, + Watched: make([]string, 0), + Watchable: datadir.WatchableModulePaths(modPath), + } + w.modules = append(w.modules, wm) + + // We watch individual dirs (instead of individual files). + // This does result in more events but fewer watched paths. + // fsnotify does not support recursive watching yet. + // See https://github.com/fsnotify/fsnotify/issues/18 + + err := w.fw.Add(modPath) + if err != nil { + return err + } + + for _, dirPath := range wm.Watchable.Dirs { + err := w.fw.Add(dirPath) + if err == nil { + wm.Watched = append(wm.Watched, dirPath) + } + } + + return nil +} + +func (w *watcher) run(ctx context.Context) { + for { + select { + case event, ok := <-w.fw.Events: + if !ok { + return + } + w.processEvent(event) + case err, ok := <-w.fw.Errors: + if !ok { + return + } + w.logger.Println("watch error:", err) + } + } +} + +func (w *watcher) processEvent(event fsnotify.Event) { + eventPath := event.Name + + if event.Op&fsnotify.Write == fsnotify.Write { + for _, mod := range w.modules { + if containsPath(mod.Watchable.ModuleManifests, eventPath) { + w.modMgr.EnqueueModuleOp(mod.Path, OpTypeParseModuleManifest) + return + } + if containsPath(mod.Watchable.PluginLockFiles, eventPath) { + w.modMgr.EnqueueModuleOp(mod.Path, OpTypeObtainSchema) + return + } + } + } + + if event.Op&fsnotify.Create == fsnotify.Create { + for _, mod := range w.modules { + if containsPath(mod.Watchable.Dirs, eventPath) { + w.fw.Add(eventPath) + mod.Watched = append(mod.Watched, eventPath) + + filepath.Walk(eventPath, func(path string, info os.FileInfo, err error) error { + if info.IsDir() { + if containsPath(mod.Watchable.Dirs, path) { + w.fw.Add(path) + mod.Watched = append(mod.Watched, path) + } + return nil + } + if containsPath(mod.Watchable.ModuleManifests, path) { + return w.modMgr.EnqueueModuleOp(mod.Path, OpTypeParseModuleManifest) + } + if containsPath(mod.Watchable.PluginLockFiles, path) { + return w.modMgr.EnqueueModuleOp(mod.Path, OpTypeObtainSchema) + } + return nil + }) + + return + } + + if containsPath(mod.Watchable.ModuleManifests, eventPath) { + w.modMgr.EnqueueModuleOp(mod.Path, OpTypeParseModuleManifest) + return + } + + if containsPath(mod.Watchable.PluginLockFiles, eventPath) { + w.modMgr.EnqueueModuleOp(mod.Path, OpTypeObtainSchema) + return + } + } + } + + if event.Op&fsnotify.Remove == fsnotify.Remove { + for modI, mod := range w.modules { + // Whole module being removed + if pathEquals(mod.Path, eventPath) { + for _, wPath := range mod.Watched { + w.fw.Remove(wPath) + } + w.fw.Remove(mod.Path) + w.modules = append(w.modules[:modI], w.modules[modI+1:]...) + return + } + + for i, wp := range mod.Watched { + if pathEquals(wp, eventPath) { + w.fw.Remove(wp) + mod.Watched = append(mod.Watched[:i], mod.Watched[i+1:]...) + return + } + } + } + } +} + +func containsPath(paths []string, path string) bool { + for _, p := range paths { + if pathEquals(p, path) { + return true + } + } + return false +} + +func (w *watcher) Start() error { + if w.watching { + w.logger.Println("watching already in progress") + return nil + } + + ctx, cancelFunc := context.WithCancel(context.Background()) + w.cancelFunc = cancelFunc + w.watching = true + + w.logger.Printf("watching for changes ...") + go w.run(ctx) + + return nil +} + +func (w *watcher) Stop() error { + if !w.watching { + return nil + } + + w.cancelFunc() + + err := w.fw.Close() + if err == nil { + w.watching = false + } + + return err +} diff --git a/internal/watcher/watcher_mock.go b/internal/terraform/module/watcher_mock.go similarity index 52% rename from internal/watcher/watcher_mock.go rename to internal/terraform/module/watcher_mock.go index 8eef0ddf6..bc5b0d9d2 100644 --- a/internal/watcher/watcher_mock.go +++ b/internal/terraform/module/watcher_mock.go @@ -1,34 +1,32 @@ -package watcher +package module import ( "log" + + "github.com/hashicorp/terraform-ls/internal/filesystem" ) func MockWatcher() WatcherFactory { - return func() (Watcher, error) { + return func(filesystem.Filesystem, ModuleManager) (Watcher, error) { return &mockWatcher{}, nil } } type mockWatcher struct{} -func (w *mockWatcher) AddChangeHook(h ChangeHook) { -} - -func (w *mockWatcher) AddPaths(paths []string) error { +func (w *mockWatcher) Start() error { return nil } - -func (w *mockWatcher) AddPath(path string) error { +func (w *mockWatcher) Stop() error { return nil } -func (w *mockWatcher) Start() error { - return nil -} +func (w *mockWatcher) SetLogger(*log.Logger) {} -func (w *mockWatcher) Stop() error { +func (w *mockWatcher) AddModule(string) error { return nil } -func (w *mockWatcher) SetLogger(*log.Logger) {} +func (w *mockWatcher) IsModuleWatched(string) bool { + return false +} diff --git a/internal/terraform/module/watcher_test.go b/internal/terraform/module/watcher_test.go new file mode 100644 index 000000000..10e461ad2 --- /dev/null +++ b/internal/terraform/module/watcher_test.go @@ -0,0 +1,117 @@ +package module + +import ( + "context" + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/hashicorp/terraform-json" + "github.com/hashicorp/terraform-ls/internal/filesystem" + "github.com/hashicorp/terraform-ls/internal/terraform/exec" + "github.com/stretchr/testify/mock" +) + +func TestWatcher_initFromScratch(t *testing.T) { + fs := filesystem.NewFilesystem() + + modPath := filepath.Join(t.TempDir(), "module") + err := os.Mkdir(modPath, 0755) + if err != nil { + t.Fatal(err) + } + + psMock := &tfjson.ProviderSchemas{ + FormatVersion: "0.1", + Schemas: map[string]*tfjson.ProviderSchema{ + "custom": {}, + }, + } + mmm := NewModuleManagerMock(&ModuleManagerMockInput{ + Logger: testLogger(), + TerraformCalls: &exec.TerraformMockCalls{ + PerWorkDir: map[string][]*mock.Call{ + modPath: { + { + Method: "ProviderSchemas", + Arguments: []interface{}{ + mock.AnythingOfType("*context.cancelCtx"), + }, + ReturnArguments: []interface{}{ + psMock, + nil, + }, + }, + }, + }, + }, + }) + ctx := context.Background() + modMgr := mmm(ctx, fs) + + w, err := NewWatcher(fs, modMgr) + if err != nil { + t.Fatal(err) + } + w.SetLogger(testLogger()) + + mod, err := modMgr.AddModule(modPath) + if err != nil { + t.Fatal(err) + } + + b := []byte(` +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 3.0" + } + } +} + +provider "aws" { + region = "us-east-1" +} + +resource "aws_vpc" "example" { + cidr_block = "10.0.0.0/16" +} +`) + err = ioutil.WriteFile(filepath.Join(modPath, "main.tf"), b, 0755) + if err != nil { + t.Fatal(err) + } + + err = w.AddModule(modPath) + if err != nil { + t.Fatal(err) + } + + err = w.Start() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + w.Stop() + }) + + err = ioutil.WriteFile(filepath.Join(modPath, ".terraform.lock.hcl"), b, 0755) + if err != nil { + t.Fatal(err) + } + + // Give watcher some time to react + time.Sleep(100 * time.Millisecond) + + ps, err := mod.ProviderSchema() + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(psMock, ps); diff != "" { + t.Fatalf("schema mismatch: %s", diff) + } +} diff --git a/internal/watcher/tracked_file.go b/internal/watcher/tracked_file.go deleted file mode 100644 index 861bf6a1a..000000000 --- a/internal/watcher/tracked_file.go +++ /dev/null @@ -1,54 +0,0 @@ -package watcher - -import ( - "crypto/sha256" - "io" - "os" - "path/filepath" -) - -func trackedFileFromPath(path string) (TrackedFile, error) { - path, err := filepath.EvalSymlinks(path) - if err != nil { - return nil, err - } - - b, err := fileSha256Sum(path) - if err != nil { - return nil, err - } - - return &trackedFile{ - path: path, - sha256sum: string(b), - }, nil -} - -type trackedFile struct { - path string - sha256sum string -} - -func (tf *trackedFile) Path() string { - return tf.path -} - -func (tf *trackedFile) Sha256Sum() string { - return tf.sha256sum -} - -func fileSha256Sum(path string) ([]byte, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer f.Close() - - h := sha256.New() - _, err = io.Copy(h, f) - if err != nil { - return nil, err - } - - return h.Sum(nil), nil -} diff --git a/internal/watcher/types.go b/internal/watcher/types.go deleted file mode 100644 index 0555bb23d..000000000 --- a/internal/watcher/types.go +++ /dev/null @@ -1,22 +0,0 @@ -package watcher - -import ( - "context" - "log" -) - -type TrackedFile interface { - Path() string - Sha256Sum() string -} - -type Watcher interface { - Start() error - Stop() error - SetLogger(logger *log.Logger) - AddPath(path string) error - AddPaths(paths []string) error - AddChangeHook(f ChangeHook) -} - -type ChangeHook func(ctx context.Context, file TrackedFile) error diff --git a/internal/watcher/watcher.go b/internal/watcher/watcher.go deleted file mode 100644 index 252859543..000000000 --- a/internal/watcher/watcher.go +++ /dev/null @@ -1,137 +0,0 @@ -package watcher - -import ( - "context" - "io/ioutil" - "log" - - "github.com/fsnotify/fsnotify" -) - -// Watcher is a wrapper around native fsnotify.Watcher -// It provides the ability to detect actual file changes -// (rather than just events that may not be changing any bytes) -type watcher struct { - fw *fsnotify.Watcher - trackedFiles map[string]TrackedFile - changeHooks []ChangeHook - logger *log.Logger - - watching bool - cancelFunc context.CancelFunc -} - -type WatcherFactory func() (Watcher, error) - -func NewWatcher() (Watcher, error) { - fw, err := fsnotify.NewWatcher() - if err != nil { - return nil, err - } - return &watcher{ - fw: fw, - logger: defaultLogger, - trackedFiles: make(map[string]TrackedFile, 0), - }, nil -} - -var defaultLogger = log.New(ioutil.Discard, "", 0) - -func (w *watcher) SetLogger(logger *log.Logger) { - w.logger = logger -} - -func (w *watcher) AddPaths(paths []string) error { - for _, p := range paths { - err := w.AddPath(p) - if err != nil { - return err - } - } - return nil -} - -func (w *watcher) AddPath(path string) error { - w.logger.Printf("adding %s for watching", path) - - tf, err := trackedFileFromPath(path) - if err != nil { - return err - } - w.trackedFiles[path] = tf - - return w.fw.Add(path) -} - -func (w *watcher) AddChangeHook(h ChangeHook) { - w.changeHooks = append(w.changeHooks, h) -} - -func (w *watcher) run(ctx context.Context) { - for { - select { - case event, ok := <-w.fw.Events: - if !ok { - return - } - - if event.Op&fsnotify.Write == fsnotify.Write { - w.logger.Printf("detected write into %s", event.Name) - oldTf := w.trackedFiles[event.Name] - newTf, err := trackedFileFromPath(event.Name) - if err != nil { - w.logger.Println("failed to track file, ignoring", err) - continue - } - w.trackedFiles[event.Name] = newTf - - if oldTf.Sha256Sum() != newTf.Sha256Sum() { - for _, h := range w.changeHooks { - err := h(ctx, newTf) - if err != nil { - w.logger.Println("change hook error:", err) - } - } - } - } - case err, ok := <-w.fw.Errors: - if !ok { - return - } - w.logger.Println("watch error:", err) - } - } -} - -// StartWatching starts to watch for changes that were added -// via AddPath(s) until Stop() is called -func (w *watcher) Start() error { - if w.watching { - w.logger.Println("watching already in progress") - return nil - } - - ctx, cancelFunc := context.WithCancel(context.Background()) - w.cancelFunc = cancelFunc - w.watching = true - - w.logger.Printf("watching for changes ...") - go w.run(ctx) - - return nil -} - -func (w *watcher) Stop() error { - if !w.watching { - return nil - } - - w.cancelFunc() - - err := w.fw.Close() - if err == nil { - w.watching = false - } - - return err -}