From 418df274d3c8188ef88eb9c36deb9199d547ba78 Mon Sep 17 00:00:00 2001 From: Tom Christensen Date: Wed, 14 Sep 2022 07:26:08 -0600 Subject: [PATCH 1/4] Adding private registry support for yarn berry Adds minimal private registry support relying fully on the fact that we will clone the repo and will use the proxy for authentication. --- .../dependabot/npm_and_yarn/file_parser.rb | 10 +++++++++ .../file_parser/lockfile_parser.rb | 7 ++++++ .../file_updater/yarn_lockfile_updater.rb | 1 - .../lib/dependabot/npm_and_yarn/helpers.rb | 22 +++++++++++++++++++ 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser.rb b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser.rb index 9c13e6d4c3c..52d2ace3f99 100644 --- a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser.rb +++ b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser.rb @@ -247,6 +247,16 @@ def source_for(name, requirement, manifest_name) manifest_name: manifest_name )&.fetch("resolved", nil) + if lockfile_parser.yarn_berry? + resolution = lockfile_parser.lockfile_details( + dependency_name: name, + requirement: requirement, + manifest_name: manifest_name + )&.fetch("resolution", nil) + package_match = resolution.match(/__archiveUrl=(?.+)/) if resolution + resolved_url = CGI.unescape(package_match.named_captures.fetch("package_url")) if package_match + end + return unless resolved_url return unless resolved_url.start_with?("http") return if resolved_url.match?(/(?\d+.\d+.\d+)/) + version_match&.named_captures&.fetch("version", nil) + end + + def self.yarn_berry? + major_version = Version.new(yarn_version).major + return true if major_version >= 2 + false + end + # Run any number of yarn commands while ensuring that `enableScripts` is # set to false. Yarn commands should _not_ be ran outside of this helper # to ensure that postinstall scripts are never executed, as they could From 7dd42262cafb580df64d1859337012c2a21e0735 Mon Sep 17 00:00:00 2001 From: Tom Christensen Date: Tue, 4 Oct 2022 00:33:20 -0600 Subject: [PATCH 2/4] A bit of code cleanup, adding feature flag checks --- .../dependabot/npm_and_yarn/file_fetcher.rb | 8 +++---- .../dependabot/npm_and_yarn/file_parser.rb | 6 ++--- .../file_parser/lockfile_parser.rb | 7 ------ .../lib/dependabot/npm_and_yarn/helpers.rb | 22 ------------------- 4 files changed, 7 insertions(+), 36 deletions(-) diff --git a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_fetcher.rb b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_fetcher.rb index a9252ae044d..3b30be3bfb1 100644 --- a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_fetcher.rb +++ b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_fetcher.rb @@ -69,15 +69,15 @@ def yarn_version return @yarn_version if defined?(@yarn_version) package = JSON.parse(package_json.content) - if (pkgmanager = package.fetch("packageManager", nil)) - get_yarn_version_from_path(pkgmanager) + if Experiments.enabled?(:yarn_berry) && (package_manager = package.fetch("packageManager", nil)) + get_yarn_version_from_package_json(package_manager) elsif yarn_lock 1 end end - def get_yarn_version_from_path(path) - version_match = path.match(/yarn@(?\d+.\d+.\d+)/) + def get_yarn_version_from_package_json(package_manager) + version_match = package_manager.match(/yarn@(?\d+.\d+.\d+)/) version_match&.named_captures&.fetch("version", nil) end diff --git a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser.rb b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser.rb index 52d2ace3f99..84e1ed743f5 100644 --- a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser.rb +++ b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser.rb @@ -247,14 +247,14 @@ def source_for(name, requirement, manifest_name) manifest_name: manifest_name )&.fetch("resolved", nil) - if lockfile_parser.yarn_berry? + if Experiments.enabled?(:yarn_berry) && resolved_url.nil? resolution = lockfile_parser.lockfile_details( dependency_name: name, requirement: requirement, manifest_name: manifest_name )&.fetch("resolution", nil) - package_match = resolution.match(/__archiveUrl=(?.+)/) if resolution - resolved_url = CGI.unescape(package_match.named_captures.fetch("package_url")) if package_match + package_match = resolution&.match(/__archiveUrl=(?.+)/) + resolved_url = CGI.unescape(package_match.named_captures.fetch("package_url", "")) if package_match end return unless resolved_url diff --git a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser/lockfile_parser.rb b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser/lockfile_parser.rb index 68a506aa170..81da602137f 100644 --- a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser/lockfile_parser.rb +++ b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser/lockfile_parser.rb @@ -35,13 +35,6 @@ def lockfile_details(dependency_name:, requirement:, manifest_name:) nil end - def yarn_berry? - yaml = YAML.safe_load(yarn_locks.first.content) - yaml.key?("__metadata") - rescue StandardError - false - end - private attr_reader :dependency_files diff --git a/npm_and_yarn/lib/dependabot/npm_and_yarn/helpers.rb b/npm_and_yarn/lib/dependabot/npm_and_yarn/helpers.rb index 1053092cff5..832de4beabd 100644 --- a/npm_and_yarn/lib/dependabot/npm_and_yarn/helpers.rb +++ b/npm_and_yarn/lib/dependabot/npm_and_yarn/helpers.rb @@ -16,28 +16,6 @@ def self.npm_version_numeric(lockfile_content) 6 end - def self.yarn_version(package_json_content, yarn_lock) - return @yarn_version if defined?(@yarn_version) - - package = JSON.parse(package_json_content) - if (package_manager = package.fetch("packageManager", nil)) - get_yarn_version_from_path(package_manager) - elsif yarn_lock - 1 - end - end - - def self.get_yarn_version(package_manager) - version_match = package_manager.match(/yarn@(?\d+.\d+.\d+)/) - version_match&.named_captures&.fetch("version", nil) - end - - def self.yarn_berry? - major_version = Version.new(yarn_version).major - return true if major_version >= 2 - false - end - # Run any number of yarn commands while ensuring that `enableScripts` is # set to false. Yarn commands should _not_ be ran outside of this helper # to ensure that postinstall scripts are never executed, as they could From 56faa9531814d18c3b5d933c92f3317936766e5a Mon Sep 17 00:00:00 2001 From: Tom Christensen Date: Tue, 4 Oct 2022 00:49:15 -0600 Subject: [PATCH 3/4] attempt to reduce complexity. --- .../lib/dependabot/npm_and_yarn/file_parser.rb | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser.rb b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser.rb index 84e1ed743f5..7e72b6eb590 100644 --- a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser.rb +++ b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser.rb @@ -241,18 +241,15 @@ def semver_version_for(name, requirement, manifest_name) def source_for(name, requirement, manifest_name) return git_source_for(requirement) if git_url?(requirement) - resolved_url = lockfile_parser.lockfile_details( + lockfile_details = lockfile_parser.lockfile_details( dependency_name: name, requirement: requirement, manifest_name: manifest_name - )&.fetch("resolved", nil) + ) + resolved_url = lockfile_details&.fetch("resolved", nil) if Experiments.enabled?(:yarn_berry) && resolved_url.nil? - resolution = lockfile_parser.lockfile_details( - dependency_name: name, - requirement: requirement, - manifest_name: manifest_name - )&.fetch("resolution", nil) + resolution = lockfile_details&.fetch("resolution", nil) package_match = resolution&.match(/__archiveUrl=(?.+)/) resolved_url = CGI.unescape(package_match.named_captures.fetch("package_url", "")) if package_match end From df667d63641bbe7400320d9e9377ce7838a1aa4f Mon Sep 17 00:00:00 2001 From: Tom Christensen Date: Tue, 4 Oct 2022 01:02:50 -0600 Subject: [PATCH 4/4] Adding back inadvertently removed whitespace. --- .../npm_and_yarn/file_updater/yarn_lockfile_updater.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater/yarn_lockfile_updater.rb b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater/yarn_lockfile_updater.rb index 7cca7f78d6a..0b7ba03f657 100644 --- a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater/yarn_lockfile_updater.rb +++ b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater/yarn_lockfile_updater.rb @@ -178,6 +178,7 @@ def requirements_changed?(dependency_name) def run_yarn_berry_subdependency_updater(yarn_lock:) dep = sub_dependencies.first update = "#{dep.name}@#{dep.version}" + Helpers.run_yarn_commands( "yarn add #{update} --mode=update-lockfile", "yarn dedupe #{dep.name} --mode=update-lockfile",