Go: Use go-get=1 trick to find git source URLs for unrecognised names

lib/dependabot/file_parsers/go/dep.rb

@@ -4,6 +4,9 @@
 
 require "dependabot/errors"
 require "dependabot/dependency"
+require "dependabot/shared_helpers"
+require "dependabot/source"
+
 require "dependabot/file_parsers/base"
 require "dependabot/utils/go/requirement"
 
@@ -80,10 +83,6 @@ def requirement_from_declaration(declaration)
         end
 
         def source_from_declaration(declaration)
-          unless declaration.is_a?(Hash)
-            raise "Unexpected dependency declaration: #{declaration}"
-          end
-
           source = declaration["source"] || declaration["name"]
 
           git_source = git_source(source)
@@ -95,6 +94,8 @@ def source_from_declaration(declaration)
               branch: declaration["branch"],
               ref: declaration["revision"] || declaration["version"]
             }
+          elsif git_declaration?(declaration)
+            raise "No git source for a git declaration!"
           else
             {
               type: "default",
@@ -120,12 +121,31 @@ def git_declaration?(declaration)
         end
 
         def git_source(path)
+          # Save a query by doing the conversion of golang.org/x names manually
           updated_path = path.gsub(%r{^golang\.org/x}, "github.com/golang")
 
           # Currently, Dependabot::Source.new will return `nil` if it can't find
           # a git SCH associated with a path. If it is ever extended to handle
           # non-git sources we'll need to add an additional check here.
-          Source.from_url(updated_path)
+          return Source.from_url(updated_path) if Source.from_url(updated_path)
+
+          # TODO: This is not robust! Instead, we should shell out to Go and use
+          # https://github.com/Masterminds/vcs.
+          uri = "https://#{path}?go-get=1"
+          response = Excon.get(
+            uri,
+            idempotent: true,
+            **SharedHelpers.excon_defaults
+          )
+
+          return unless response.status == 200
+
+          response.body.scan(Dependabot::Source::SOURCE_REGEX) do
+            source_url = Regexp.last_match.to_s
+            return Source.from_url(source_url)
+          end
+
+          nil
         end
 
         def parsed_file(file)

spec/dependabot/file_parsers/go/dep_spec.rb

@@ -33,6 +33,14 @@
     )
   end
 
+  before do
+    stub_request(:get, "https://golang.org/x/text?go-get=1").
+      to_return(
+        status: 200,
+        body: fixture("go", "repo_responses", "golang_org_text.html")
+      )
+  end
+
   describe "parse" do
     subject(:dependencies) { parser.parse }
 
@@ -178,30 +186,42 @@
             )
           end
         end
-      end
 
-      describe "a dependency with an unrecognised name" do
-        let(:manifest_fixture_name) { "unknown_source.toml" }
-        let(:lockfile_fixture_name) { "unknown_source.lock" }
-        subject(:dependency) do
-          dependencies.find { |d| d.name == "unknownhost.com/dgrijalva/jwt-go" }
-        end
+        describe "with a proxy host in the name" do
+          let(:manifest_fixture_name) { "proxy_git_source.toml" }
+          let(:lockfile_fixture_name) { "proxy_git_source.lock" }
+          subject(:dependency) do
+            dependencies.find { |d| d.name == "k8s.io/apimachinery" }
+          end
 
-        it "has the right details" do
-          expect(dependency).to be_a(Dependabot::Dependency)
-          expect(dependency.name).to eq("unknownhost.com/dgrijalva/jwt-go")
-          expect(dependency.version).to eq("3.2.0")
-          expect(dependency.requirements).to eq(
-            [{
-              requirement: "^3.2.0",
-              file: "Gopkg.toml",
-              groups: [],
-              source: {
-                type: "default",
-                source: "unknownhost.com/dgrijalva/jwt-go"
-              }
-            }]
-          )
+          before do
+            stub_request(:get, "https://k8s.io/apimachinery?go-get=1").
+              to_return(
+                status: 200,
+                body:
+                  fixture("go", "repo_responses", "k8s_io_apimachinery.html")
+              )
+          end
+
+          it "has the right details" do
+            expect(dependency).to be_a(Dependabot::Dependency)
+            expect(dependency.name).to eq("k8s.io/apimachinery")
+            expect(dependency.version).
+              to eq("cbafd24d5796966031ae904aa88e2436a619ae8a")
+            expect(dependency.requirements).to eq(
+              [{
+                requirement: nil,
+                file: "Gopkg.toml",
+                groups: [],
+                source: {
+                  type: "git",
+                  url: "https://github.com/kubernetes/apimachinery",
+                  branch: "master",
+                  ref: nil
+                }
+              }]
+            )
+          end
         end
       end
     end

spec/fixtures/go/gopkg_locks/proxy_git_source.lock

@@ -0,0 +1,132 @@
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[[projects]]
+  digest = "1:6e73003ecd35f4487a5e88270d3ca0a81bc80dc88053ac7e4dcfec5fba30d918"
+  name = "github.com/gogo/protobuf"
+  packages = [
+    "proto",
+    "sortkeys",
+  ]
+  pruneopts = ""
+  revision = "636bf0302bc95575d69441b25a2603156ffdddf1"
+  version = "v1.1.1"
+
+[[projects]]
+  branch = "master"
+  digest = "1:107b233e45174dbab5b1324201d092ea9448e58243ab9f039e4c0f332e121e3a"
+  name = "github.com/golang/glog"
+  packages = ["."]
+  pruneopts = ""
+  revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998"
+
+[[projects]]
+  branch = "master"
+  digest = "1:754f77e9c839b24778a4b64422236d38515301d2baeb63113aa3edc42e6af692"
+  name = "github.com/google/gofuzz"
+  packages = ["."]
+  pruneopts = ""
+  revision = "24818f796faf91cd76ec7bddd72458fbced7a6c1"
+
+[[projects]]
+  digest = "1:d5c1692d62c22b9ff5248d1c5f3850d06f7bc850f9ae6eec15c6601f774eef4f"
+  name = "github.com/jinzhu/gorm"
+  packages = ["."]
+  pruneopts = ""
+  revision = "5174cc5c242a728b435ea2be8a2f7f998e15429b"
+  version = "v1.0"
+
+[[projects]]
+  branch = "master"
+  digest = "1:d9a7385b84d8187fd94e0357045c6fa1147ca94caa56fdd539336c7c102fc728"
+  name = "github.com/jinzhu/inflection"
+  packages = ["."]
+  pruneopts = ""
+  revision = "04140366298a54a039076d798123ffa108fff46c"
+
+[[projects]]
+  branch = "master"
+  digest = "1:96d281cfaaa12ac602772da38ac85f00d59b1d3aa7bfe69d8ba334d6ee41e3e6"
+  name = "golang.org/x/net"
+  packages = [
+    "http/httpguts",
+    "http2",
+    "http2/hpack",
+    "idna",
+  ]
+  pruneopts = ""
+  revision = "3673e40ba22529d22c3fd7c93e97b0ce50fa7bdd"
+
+[[projects]]
+  digest = "1:5acd3512b047305d49e8763eef7ba423901e85d5dd2fd1e71778a0ea8de10bd4"
+  name = "golang.org/x/text"
+  packages = [
+    "collate",
+    "collate/build",
+    "internal/colltab",
+    "internal/gen",
+    "internal/tag",
+    "internal/triegen",
+    "internal/ucd",
+    "language",
+    "secure/bidirule",
+    "transform",
+    "unicode/bidi",
+    "unicode/cldr",
+    "unicode/norm",
+    "unicode/rangetable",
+  ]
+  pruneopts = ""
+  revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
+  version = "v0.3.0"
+
+[[projects]]
+  digest = "1:75fb3fcfc73a8c723efde7777b40e8e8ff9babf30d8c56160d01beffea8a95a6"
+  name = "gopkg.in/inf.v0"
+  packages = ["."]
+  pruneopts = ""
+  revision = "d2d2541c53f18d2a059457998ce2876cc8e67cbf"
+  version = "v0.9.1"
+
+[[projects]]
+  branch = "master"
+  digest = "1:c4a1a585962341863994f38e614b990c55602f81ff6807eb042eae0ddf158b44"
+  name = "k8s.io/apimachinery"
+  packages = [
+    "pkg/api/meta",
+    "pkg/api/resource",
+    "pkg/apis/meta/v1",
+    "pkg/apis/meta/v1beta1",
+    "pkg/conversion",
+    "pkg/conversion/queryparams",
+    "pkg/fields",
+    "pkg/labels",
+    "pkg/runtime",
+    "pkg/runtime/schema",
+    "pkg/selection",
+    "pkg/types",
+    "pkg/util/errors",
+    "pkg/util/intstr",
+    "pkg/util/json",
+    "pkg/util/naming",
+    "pkg/util/net",
+    "pkg/util/runtime",
+    "pkg/util/sets",
+    "pkg/util/validation",
+    "pkg/util/validation/field",
+    "pkg/util/wait",
+    "pkg/watch",
+    "third_party/forked/golang/reflect",
+  ]
+  pruneopts = ""
+  revision = "cbafd24d5796966031ae904aa88e2436a619ae8a"
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  input-imports = [
+    "github.com/jinzhu/gorm",
+    "k8s.io/apimachinery/pkg/api/meta",
+  ]
+  solver-name = "gps-cdcl"
+  solver-version = 1

spec/fixtures/go/gopkg_tomls/proxy_git_source.toml

@@ -0,0 +1,3 @@
+[[constraint]]
+  branch = "master"
+  name = "k8s.io/apimachinery"

spec/fixtures/go/repo_responses/golang_org_text.html

@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html>
+    <head>
+        <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
+        <meta name="go-import" content="golang.org/x/text git https://go.googlesource.com/text">
+        <meta name="go-source" content="golang.org/x/text https://github.com/golang/text/ https://github.com/golang/text/tree/master{/dir} https://github.com/golang/text/blob/master{/dir}/{file}#L{line}">
+        <meta http-equiv="refresh" content="0; url=https://godoc.org/golang.org/x/text">
+    </head>
+    <body>
+        Nothing to see here; <a href="https://godoc.org/golang.org/x/text">move along</a>.
+    </body>
+</html>

spec/fixtures/go/repo_responses/k8s_io_apimachinery.html

@@ -0,0 +1,10 @@
+<html>
+    <head>
+        <meta name="go-import" content="k8s.io/apimachinery git https://github.com/kubernetes/apimachinery">
+        <meta name="go-source" content="k8s.io/apimachinery
+            https://github.com/kubernetes/apimachinery
+            https://github.com/kubernetes/apimachinery/tree/master{/dir}
+            https://github.com/kubernetes/apimachinery/blob/master{/dir}/{file}#L{line}">
+    </head>
+</html>
+