Merge pull request #4434 from dependabot/mctofu/go-mod-list-versions

Use `go list -m -versions` to determine available versions of a go module

common/lib/dependabot/shared_helpers.rb

@@ -280,10 +280,10 @@ def self.reset_global_git_config(backup_path)
       FileUtils.mv(backup_path, GIT_CONFIG_GLOBAL_PATH)
     end
 
-    def self.run_shell_command(command, allow_unsafe_shell_command: false)
+    def self.run_shell_command(command, allow_unsafe_shell_command: false, env: {})
       start = Time.now
       cmd = allow_unsafe_shell_command ? command : escape_command(command)
-      stdout, process = Open3.capture2e(cmd)
+      stdout, process = Open3.capture2e(env || {}, cmd)
       time_taken = Time.now - start
 
       # Raise an error with the output from the shell session if the

common/spec/dependabot/shared_helpers_spec.rb

@@ -187,9 +187,10 @@ def existing_tmp_folders
 
   describe ".run_shell_command" do
     let(:command) { File.join(spec_root, "helpers/test/run_bash") + " output" }
+    let(:env) { nil }
 
     subject(:run_shell_command) do
-      Dependabot::SharedHelpers.run_shell_command(command)
+      Dependabot::SharedHelpers.run_shell_command(command, env: env)
     end
 
     context "when the subprocess is successful" do
@@ -221,6 +222,14 @@ def existing_tmp_folders
       end
     end
 
+    context "with an environment variable" do
+      let(:env) { { "TEST_ENV" => "prefix:" } }
+
+      it "is available to the command" do
+        expect(run_shell_command).to eq("prefix:output\n")
+      end
+    end
+
     context "when the subprocess exits" do
       let(:command) { File.join(spec_root, "helpers/test/error_bash") }
 

common/spec/helpers/test/run_bash

@@ -2,4 +2,4 @@
 
 set -e
 
-echo "$@"
+echo "$TEST_ENV$@"

go_modules/helpers/go.mod

@@ -2,8 +2,4 @@ module github.com/dependabot/dependabot-core/go_modules/helpers
 
 go 1.16
 
-require (
-	github.com/Masterminds/vcs v1.13.1
-	github.com/dependabot/gomodules-extracted v1.4.2
-	golang.org/x/mod v0.5.1
-)
+require github.com/Masterminds/vcs v1.13.1

go_modules/helpers/go.sum

@@ -1,20 +1,2 @@
 github.com/Masterminds/vcs v1.13.1 h1:NL3G1X7/7xduQtA2sJLpVpfHTNBALVNSjob6KEjPXNQ=
 github.com/Masterminds/vcs v1.13.1/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
-github.com/dependabot/gomodules-extracted v1.4.2 h1:3IxvHARuuSojSNUHguc6kzWgs+uQN3fdRCowJMU1kDE=
-github.com/dependabot/gomodules-extracted v1.4.2/go.mod h1:cpzrmDX1COyhSDQXHfkRMw0STb0vmguBFqmrkr51h1I=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
-golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e h1:aZzprAO9/8oim3qStq3wc1Xuxx4QmAGriC4VU4ojemQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

go_modules/helpers/main.go

@@ -7,7 +7,6 @@ import (
 	"os"
 
 	"github.com/dependabot/dependabot-core/go_modules/helpers/importresolver"
-	"github.com/dependabot/dependabot-core/go_modules/helpers/updatechecker"
 )
 
 type HelperParams struct {
@@ -32,10 +31,6 @@ func main() {
 		funcErr error
 	)
 	switch helperParams.Function {
-	case "getVersions":
-		var args updatechecker.Args
-		parseArgs(helperParams.Args, &args)
-		funcOut, funcErr = updatechecker.GetVersions(&args)
 	case "getVcsRemoteForImport":
 		var args importresolver.Args
 		parseArgs(helperParams.Args, &args)

go_modules/lib/dependabot/go_modules/update_checker/latest_version_finder.rb

@@ -20,9 +20,11 @@ class LatestVersionFinder
           /404 Not Found/,
           /Repository not found/,
           /unrecognized import path/,
+          /malformed module path/,
           # (Private) module could not be fetched
           /module .*: git ls-remote .*: exit status 128/m.freeze
         ].freeze
+        INVALID_VERSION_REGEX = /version "[^"]+" invalid/m.freeze
         PSEUDO_VERSION_REGEX = /\b\d{14}-[0-9a-f]{12}$/.freeze
 
         def initialize(dependency:, dependency_files:, credentials:,
@@ -73,23 +75,22 @@ def fetch_lowest_security_fix_version
         def available_versions
           SharedHelpers.in_a_temporary_directory do
             SharedHelpers.with_git_configured(credentials: credentials) do
-              File.write("go.mod", go_mod.content)
+              manifest = parse_manifest
+
+              # Set up an empty go.mod so 'go list -m' won't attempt to download dependencies. This
+              # appears to be a side effect of operating with GOPRIVATE=*. We'll retain any exclude
+              # directives to omit those versions.
+              File.write("go.mod", "module dummy\n")
+              manifest["Exclude"]&.each do |r|
+                SharedHelpers.run_shell_command("go mod edit -exclude=#{r['Path']}@#{r['Version']}")
+              end
 
               # Turn off the module proxy for now, as it's causing issues with
               # private git dependencies
               env = { "GOPRIVATE" => "*" }
 
-              version_strings = SharedHelpers.run_helper_subprocess(
-                command: NativeHelpers.helper_path,
-                env: env,
-                function: "getVersions",
-                args: {
-                  dependency: {
-                    name: dependency.name,
-                    version: "v" + dependency.version
-                  }
-                }
-              )
+              versions_json = SharedHelpers.run_shell_command("go list -m -versions -json #{dependency.name}", env: env)
+              version_strings = JSON.parse(versions_json)["Versions"]
 
               return [version_class.new(dependency.version)] if version_strings.nil?
 
@@ -108,6 +109,8 @@ def available_versions
         def handle_subprocess_error(error)
           if RESOLVABILITY_ERROR_REGEXES.any? { |rgx| error.message =~ rgx }
             ResolvabilityErrors.handle(error.message, credentials: credentials)
+          elsif INVALID_VERSION_REGEX =~ error.message
+            raise Dependabot::DependencyFileNotResolvable, error.message
           end
 
           raise
@@ -123,6 +126,15 @@ def go_mod
           @go_mod ||= dependency_files.find { |f| f.name == "go.mod" }
         end
 
+        def parse_manifest
+          SharedHelpers.in_a_temporary_directory do
+            File.write("go.mod", go_mod.content)
+            json = SharedHelpers.run_shell_command("go mod edit -json")
+
+            JSON.parse(json) || {}
+          end
+        end
+
         def filter_prerelease_versions(versions_array)
           return versions_array if wants_prerelease?
 

go_modules/spec/dependabot/go_modules/update_checker/latest_version_finder_spec.rb

@@ -66,11 +66,11 @@ module foobar
     end
 
     context "when already on the latest version" do
-      let(:dependency_name) { "github.com/dependabot-fixtures/go-modules-lib/v2" }
-      let(:dependency_version) { "2.0.0" }
+      let(:dependency_name) { "github.com/dependabot-fixtures/go-modules-lib/v3" }
+      let(:dependency_version) { "3.0.0" }
 
       it "returns the current version" do
-        expect(finder.latest_version).to eq(Dependabot::GoModules::Version.new("2.0.0"))
+        expect(finder.latest_version).to eq(Dependabot::GoModules::Version.new("3.0.0"))
       end
     end
 
@@ -186,6 +186,34 @@ module foobar
       end
     end
 
+    context "when the dependency's major version is invalid because it's not specified in its go.mod" do
+      let(:dependency_name) { "github.com/dependabot-fixtures/go-modules-lib/v2" }
+      let(:dependency_version) { "2.0.0" }
+
+      it "raises a DependencyFileNotResolvable error" do
+        error_class = Dependabot::DependencyFileNotResolvable
+        expect { finder.latest_version }.
+          to raise_error(error_class) do |error|
+          expect(error.message).to include("github.com/dependabot-fixtures/go-modules-lib/v2")
+          expect(error.message).to include("version \"v2.0.0\" invalid")
+        end
+      end
+    end
+
+    context "when the dependency's major version is invalid because not properly imported" do
+      let(:dependency_name) { "github.com/dependabot-fixtures/go-modules-lib" }
+      let(:dependency_version) { "3.0.0" }
+
+      it "raises a DependencyFileNotResolvable error" do
+        error_class = Dependabot::DependencyFileNotResolvable
+        expect { finder.latest_version }.
+          to raise_error(error_class) do |error|
+          expect(error.message).to include("github.com/dependabot-fixtures/go-modules-lib")
+          expect(error.message).to include("version \"v3.0.0\" invalid")
+        end
+      end
+    end
+
     context "when the module is unreachable" do
       let(:dependency_files) { [go_mod] }
       let(:dependency_name) { "github.com/dependabot-fixtures/go-modules-private" }
@@ -212,7 +240,7 @@ module foobar
       # latest release v1.0.1 is retracted
       let(:dependency_name) { "github.com/dependabot-fixtures/go-modules-retracted" }
 
-      pending "doesn't return the retracted version" do
+      it "doesn't return the retracted version" do
         expect(finder.latest_version).to eq(Dependabot::GoModules::Version.new("1.0.0"))
       end
     end