diff --git a/go_modules/Dockerfile b/go_modules/Dockerfile index b2fe5de260f4..961412e23463 100644 --- a/go_modules/Dockerfile +++ b/go_modules/Dockerfile @@ -17,10 +17,6 @@ RUN cd /tmp \ && tar -xzf go-${TARGETARCH}.tar.gz -C /opt \ && rm go-${TARGETARCH}.tar.gz -ENV DEPENDABOT_NATIVE_HELPERS_PATH="/opt" - -COPY go_modules/helpers /opt/go_modules/helpers -RUN bash /opt/go_modules/helpers/build - USER dependabot +# TODO: can this be deleted? I think no, it needs to stay for writing to the go module cache... COPY --chown=dependabot:dependabot go_modules /home/dependabot/go_modules diff --git a/go_modules/README.md b/go_modules/README.md index 178dfe922bc5..f0bcb9a1a61e 100644 --- a/go_modules/README.md +++ b/go_modules/README.md @@ -4,18 +4,12 @@ Go modules support for [`dependabot-core`][core-repo]. ### Running locally -1. Install native helpers - ``` - $ export DEPENDABOT_NATIVE_HELPERS_PATH=$PWD/helpers/install-dir - $ helpers/build - ``` - -2. Install Ruby dependencies +1. Install Ruby dependencies ``` $ bundle install ``` -3. Run tests +2. Run tests ``` $ bundle exec rspec spec ``` diff --git a/go_modules/dependabot-go_modules.gemspec b/go_modules/dependabot-go_modules.gemspec index 6021a2a31559..73aa0f6e2211 100644 --- a/go_modules/dependabot-go_modules.gemspec +++ b/go_modules/dependabot-go_modules.gemspec @@ -35,5 +35,5 @@ Gem::Specification.new do |spec| next unless File.exist?("../.gitignore") - spec.files += `git -C #{__dir__} ls-files lib helpers -z`.split("\x0") + spec.files += `git -C #{__dir__} ls-files lib -z`.split("\x0") end diff --git a/go_modules/helpers/Makefile b/go_modules/helpers/Makefile deleted file mode 100644 index 1d5994f70054..000000000000 --- a/go_modules/helpers/Makefile +++ /dev/null @@ -1,9 +0,0 @@ -.PHONY = all - -all: darwin linux - -darwin: - GOOS=darwin GOARCH=amd64 go build -o go-helpers.darwin64 . - -linux: - GOOS=linux GOARCH=amd64 go build -o go-helpers.linux64 . diff --git a/go_modules/helpers/build b/go_modules/helpers/build deleted file mode 100755 index 0833055ed06b..000000000000 --- a/go_modules/helpers/build +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -set -e - -if [ -z "$DEPENDABOT_NATIVE_HELPERS_PATH" ]; then - echo "Unable to build, DEPENDABOT_NATIVE_HELPERS_PATH is not set" - exit 1 -fi - -install_dir="$DEPENDABOT_NATIVE_HELPERS_PATH/go_modules" - -if ! [[ "$install_dir" =~ ^/ ]]; then - echo "$install_dir must be an absolute path" - exit 1 -fi - -if [ ! -d "$install_dir/bin" ]; then - mkdir -p "$install_dir/bin" -fi - -helpers_dir="$(dirname "${BASH_SOURCE[0]}")" -cd "$helpers_dir" - -os="$(uname -s | tr '[:upper:]' '[:lower:]')" -echo "building $install_dir/bin/helper" - -GOOS="$os" GOARCH=amd64 go build -o "$install_dir/bin/helper" . -go clean -cache -modcache diff --git a/go_modules/helpers/go.mod b/go_modules/helpers/go.mod deleted file mode 100644 index 56db04ad0be4..000000000000 --- a/go_modules/helpers/go.mod +++ /dev/null @@ -1,5 +0,0 @@ -module github.com/dependabot/dependabot-core/go_modules/helpers - -go 1.20 - -require github.com/Masterminds/vcs v1.13.3 diff --git a/go_modules/helpers/go.sum b/go_modules/helpers/go.sum deleted file mode 100644 index 89365af4e39f..000000000000 --- a/go_modules/helpers/go.sum +++ /dev/null @@ -1,2 +0,0 @@ -github.com/Masterminds/vcs v1.13.3 h1:IIA2aBdXvfbIM+yl/eTnL4hb1XwdpvuQLglAix1gweE= -github.com/Masterminds/vcs v1.13.3/go.mod h1:TiE7xuEjl1N4j016moRd6vezp6e6Lz23gypeXfzXeW8= diff --git a/go_modules/helpers/importresolver/main.go b/go_modules/helpers/importresolver/main.go deleted file mode 100644 index 938980edbeff..000000000000 --- a/go_modules/helpers/importresolver/main.go +++ /dev/null @@ -1,34 +0,0 @@ -package importresolver - -import ( - "io/ioutil" - "strings" - - "github.com/Masterminds/vcs" -) - -type Args struct { - Import string -} - -func VCSRemoteForImport(args *Args) (interface{}, error) { - remote := args.Import - scheme := strings.Split(remote, ":")[0] - switch scheme { - case "http", "https": - default: - remote = "https://" + remote - } - - local, err := ioutil.TempDir("", "unused-vcs-local-dir") - if err != nil { - return nil, err - } - - repo, err := vcs.NewRepo(remote, local) - if err != nil { - return nil, err - } - - return repo.Remote(), nil -} diff --git a/go_modules/helpers/main.go b/go_modules/helpers/main.go deleted file mode 100644 index 4f8d6e8747d4..000000000000 --- a/go_modules/helpers/main.go +++ /dev/null @@ -1,67 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "log" - "os" - - "github.com/dependabot/dependabot-core/go_modules/helpers/importresolver" -) - -type HelperParams struct { - Function string `json:"function"` - Args json.RawMessage `json:"args"` -} - -type Output struct { - Error string `json:"error,omitempty"` - Result interface{} `json:"result,omitempty"` -} - -func main() { - d := json.NewDecoder(os.Stdin) - helperParams := &HelperParams{} - if err := d.Decode(helperParams); err != nil { - abort(err) - } - - var ( - funcOut interface{} - funcErr error - ) - switch helperParams.Function { - case "getVcsRemoteForImport": - var args importresolver.Args - parseArgs(helperParams.Args, &args) - funcOut, funcErr = importresolver.VCSRemoteForImport(&args) - default: - abort(fmt.Errorf("unrecognised function '%s'", helperParams.Function)) - } - - if funcErr != nil { - abort(funcErr) - } - - output(&Output{Result: funcOut}) -} - -func parseArgs(data []byte, args interface{}) { - if err := json.Unmarshal(data, args); err != nil { - abort(err) - } -} - -func output(o *Output) { - bytes, jsonErr := json.Marshal(o) - if jsonErr != nil { - log.Fatal(jsonErr) - } - - os.Stdout.Write(bytes) -} - -func abort(err error) { - output(&Output{Error: err.Error()}) - os.Exit(1) -} diff --git a/go_modules/lib/dependabot/go_modules/file_parser.rb b/go_modules/lib/dependabot/go_modules/file_parser.rb index 023010f8c5ce..6f7a1813baa9 100644 --- a/go_modules/lib/dependabot/go_modules/file_parser.rb +++ b/go_modules/lib/dependabot/go_modules/file_parser.rb @@ -3,7 +3,6 @@ require "open3" require "dependabot/dependency" require "dependabot/file_parsers/base/dependency_set" -require "dependabot/go_modules/path_converter" require "dependabot/go_modules/replace_stubber" require "dependabot/errors" require "dependabot/file_parsers" diff --git a/go_modules/lib/dependabot/go_modules/file_updater/go_mod_updater.rb b/go_modules/lib/dependabot/go_modules/file_updater/go_mod_updater.rb index 6ffabe89186a..8ce5e2472c04 100644 --- a/go_modules/lib/dependabot/go_modules/file_updater/go_mod_updater.rb +++ b/go_modules/lib/dependabot/go_modules/file_updater/go_mod_updater.rb @@ -4,7 +4,6 @@ require "dependabot/errors" require "dependabot/logger" require "dependabot/go_modules/file_updater" -require "dependabot/go_modules/native_helpers" require "dependabot/go_modules/replace_stubber" require "dependabot/go_modules/resolvability_errors" diff --git a/go_modules/lib/dependabot/go_modules/metadata_finder.rb b/go_modules/lib/dependabot/go_modules/metadata_finder.rb index 037d85431d13..3d6d9db64bbe 100644 --- a/go_modules/lib/dependabot/go_modules/metadata_finder.rb +++ b/go_modules/lib/dependabot/go_modules/metadata_finder.rb @@ -2,7 +2,6 @@ require "dependabot/metadata_finders" require "dependabot/metadata_finders/base" -require "dependabot/go_modules/path_converter" module Dependabot module GoModules @@ -10,9 +9,37 @@ class MetadataFinder < Dependabot::MetadataFinders::Base private def look_up_source - url = Dependabot::GoModules::PathConverter.git_url_for_path(dependency.name) + look_up_source_using_go_list(dependency.name) + end + + # TODO should this func be moved to lib/dependabot/go_modules/metadata_finder/ subdir? + # I'm unclear when we organize things into a subfolder rather than inlining... + def look_up_source_using_go_list(path) + debugger + # Save a query by manually converting golang.org/x names + import_path = path.gsub(%r{^golang\.org/x}, "github.com/golang") + + # TODO WIP, this command isn't quite right I'm reading though `go help list` to grok it + command = + "go list -m -f '{{.Origin}}' " + import_path + command = SharedHelpers.escape_command(command) + + stdout, stderr, status = Open3.capture3(environment, command) + handle_subprocess_error(stderr) unless status.success? + + debugger + + url = stdout # TODO extract URL for returned result Source.from_url(url) if url end + + def handle_subprocess_error(stderr) + # As we discover errors custom to go list, add handling for them here. + # See go_mod_updater.rb#handle_subprocess_error for example + + # We don't know what happened so we raise a generic error + msg = stderr.lines.last(10).join.strip + raise Dependabot::DependabotError, msg + end end end end diff --git a/go_modules/lib/dependabot/go_modules/native_helpers.rb b/go_modules/lib/dependabot/go_modules/native_helpers.rb deleted file mode 100644 index 676a783c00d0..000000000000 --- a/go_modules/lib/dependabot/go_modules/native_helpers.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -module Dependabot - module GoModules - module NativeHelpers - def self.helper_path - clean_path(File.join(native_helpers_root, "go_modules/bin/helper")) - end - - def self.native_helpers_root - default_path = File.join(__dir__, "../../../helpers/install-dir") - ENV.fetch("DEPENDABOT_NATIVE_HELPERS_PATH", default_path) - end - - def self.clean_path(path) - Pathname.new(path).cleanpath.to_path - end - end - end -end diff --git a/go_modules/lib/dependabot/go_modules/path_converter.rb b/go_modules/lib/dependabot/go_modules/path_converter.rb deleted file mode 100644 index dc073992a6c1..000000000000 --- a/go_modules/lib/dependabot/go_modules/path_converter.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -require "dependabot/go_modules/native_helpers" - -module Dependabot - module GoModules - module PathConverter - def self.git_url_for_path(path) - # Save a query by manually converting golang.org/x names - import_path = path.gsub(%r{^golang\.org/x}, "github.com/golang") - - SharedHelpers.run_helper_subprocess( - command: NativeHelpers.helper_path, - function: "getVcsRemoteForImport", - args: { import: import_path } - ) - end - end - end -end diff --git a/go_modules/lib/dependabot/go_modules/update_checker.rb b/go_modules/lib/dependabot/go_modules/update_checker.rb index 3f6ac0dbb11d..2329a8a493ef 100644 --- a/go_modules/lib/dependabot/go_modules/update_checker.rb +++ b/go_modules/lib/dependabot/go_modules/update_checker.rb @@ -4,7 +4,6 @@ require "dependabot/update_checkers/base" require "dependabot/shared_helpers" require "dependabot/errors" -require "dependabot/go_modules/native_helpers" require "dependabot/go_modules/version" module Dependabot diff --git a/go_modules/spec/dependabot/go_modules/metadata_finder_spec.rb b/go_modules/spec/dependabot/go_modules/metadata_finder_spec.rb index 7a8b1e28d911..d662b7840b35 100644 --- a/go_modules/spec/dependabot/go_modules/metadata_finder_spec.rb +++ b/go_modules/spec/dependabot/go_modules/metadata_finder_spec.rb @@ -63,4 +63,46 @@ it { is_expected.to eq("https://github.com/satori/go.uuid") } end end + + describe "#look_up_source_using_go_list" do + subject { described_class.look_up_source_using_go_list(path) } + + let(:path) { "gopkg.in/guregu/null.v3" } + + context "with a path that is immediately recognisable as a git source" do + let(:path) { "github.com/drewolson/testflight" } + it { is_expected.to eq("https://github.com/drewolson/testflight") } + end + + context "with a golang.org path" do + let(:path) { "golang.org/x/tools" } + it { is_expected.to eq("https://github.com/golang/tools") } + end + + context "with a path that ends in .git" do + let(:path) { "git.fd.io/govpp.git" } + it { is_expected.to eq("https://git.fd.io/govpp.git") } + end + + context "with a vanity URL that needs to be fetched" do + let(:path) { "k8s.io/apimachinery" } + it { is_expected.to eq("https://github.com/kubernetes/apimachinery") } + end + + xcontext "with a vanity URL that redirects" do + let(:path) { "code.cloudfoundry.org/bytefmt" } + it { is_expected.to eq("https://github.com/cloudfoundry/bytefmt") } + end + + context "with a vanity URL that 404s, but is otherwise valid" do + let(:path) { "gonum.org/v1/gonum" } + it { is_expected.to eq("https://github.com/gonum/gonum") } + end + + context "with a path that already includes a scheme" do + let(:path) { "https://github.com/drewolson/testflight" } + it { is_expected.to eq("https://github.com/drewolson/testflight") } + end + end + end diff --git a/go_modules/spec/dependabot/go_modules/path_converter_spec.rb b/go_modules/spec/dependabot/go_modules/path_converter_spec.rb deleted file mode 100644 index 53599d9c371d..000000000000 --- a/go_modules/spec/dependabot/go_modules/path_converter_spec.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -require "spec_helper" -require "dependabot/go_modules/path_converter" - -RSpec.describe Dependabot::GoModules::PathConverter do - describe ".git_url_for_path" do - subject { described_class.git_url_for_path(path) } - - let(:path) { "gopkg.in/guregu/null.v3" } - - context "with a path that is immediately recognisable as a git source" do - let(:path) { "github.com/drewolson/testflight" } - it { is_expected.to eq("https://github.com/drewolson/testflight") } - end - - context "with a golang.org path" do - let(:path) { "golang.org/x/tools" } - it { is_expected.to eq("https://github.com/golang/tools") } - end - - context "with a path that ends in .git" do - let(:path) { "git.fd.io/govpp.git" } - it { is_expected.to eq("https://git.fd.io/govpp.git") } - end - - context "with a vanity URL that needs to be fetched" do - let(:path) { "k8s.io/apimachinery" } - it { is_expected.to eq("https://github.com/kubernetes/apimachinery") } - end - - xcontext "with a vanity URL that redirects" do - let(:path) { "code.cloudfoundry.org/bytefmt" } - it { is_expected.to eq("https://github.com/cloudfoundry/bytefmt") } - end - - context "with a vanity URL that 404s, but is otherwise valid" do - let(:path) { "gonum.org/v1/gonum" } - it { is_expected.to eq("https://github.com/gonum/gonum") } - end - - context "with a path that already includes a scheme" do - let(:path) { "https://github.com/drewolson/testflight" } - it { is_expected.to eq("https://github.com/drewolson/testflight") } - end - end -end diff --git a/go_modules/spec/dependabot/go_modules/update_checker/latest_version_finder_spec.rb b/go_modules/spec/dependabot/go_modules/update_checker/latest_version_finder_spec.rb index b13e01fd435d..d7d469814310 100644 --- a/go_modules/spec/dependabot/go_modules/update_checker/latest_version_finder_spec.rb +++ b/go_modules/spec/dependabot/go_modules/update_checker/latest_version_finder_spec.rb @@ -3,7 +3,6 @@ require "spec_helper" require "dependabot/dependency" require "dependabot/dependency_file" -require "dependabot/go_modules/native_helpers" require "dependabot/go_modules/update_checker/latest_version_finder" RSpec.describe Dependabot::GoModules::UpdateChecker::LatestVersionFinder do