Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Bundler] FileParser: Supports multiple gemfiles #3262

Closed
wants to merge 1 commit into from
Closed

[Bundler] FileParser: Supports multiple gemfiles #3262

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 76 additions & 48 deletions bundler/lib/dependabot/bundler/file_parser.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,23 +45,25 @@ def git_source?(dependencies)
def gemfile_dependencies
dependencies = DependencySet.new

return dependencies unless gemfile
return dependencies unless gemfiles.any?

[gemfile, *evaled_gemfiles].each do |file|
parsed_gemfile.each do |dep|
all_gemfiles = gemfiles + evaled_gemfiles

all_gemfiles.each do |gemfile|
parsed_gemfile(gemfile).each do |dep|
gemfile_declaration_finder =
GemfileDeclarationFinder.new(dependency: dep, gemfile: file)
GemfileDeclarationFinder.new(dependency: dep, gemfile: gemfile)
next unless gemfile_declaration_finder.gemfile_includes_dependency?

dependencies <<
Dependency.new(
name: dep.fetch("name"),
version: dependency_version(dep.fetch("name"))&.to_s,
version: dependency_version(dep.fetch("name"), lockfile(gemfile))&.to_s,
requirements: [{
requirement: gemfile_declaration_finder.enhanced_req_string,
groups: dep.fetch("groups").map(&:to_sym),
source: dep.fetch("source")&.transform_keys(&:to_sym),
file: file.name
file: gemfile.name
}],
package_manager: "bundler"
)
Expand All @@ -71,15 +73,18 @@ def gemfile_dependencies
dependencies
end

# TODO: How to find the lockfile matching the gemspecs?
def gemspec_dependencies
dependencies = DependencySet.new

fallback_lockfile = lockfiles.first

gemspecs.each do |gemspec|
parsed_gemspec(gemspec).each do |dependency|
dependencies <<
Dependency.new(
name: dependency.fetch("name"),
version: dependency_version(dependency.fetch("name"))&.to_s,
version: dependency_version(dependency.fetch("name"), fallback_lockfile)&.to_s,
requirements: [{
requirement: dependency.fetch("requirement").to_s,
groups: if dependency.fetch("type") == "runtime"
Expand All @@ -101,41 +106,44 @@ def gemspec_dependencies
def lockfile_dependencies
dependencies = DependencySet.new

return dependencies unless lockfile
return dependencies unless lockfiles.any?

# Create a DependencySet where each element has no requirement. Any
# requirements will be added when combining the DependencySet with
# other DependencySets.
parsed_lockfile.specs.each do |dependency|
next if dependency.source.is_a?(::Bundler::Source::Path)

dependencies <<
Dependency.new(
name: dependency.name,
version: dependency_version(dependency.name)&.to_s,
requirements: [],
package_manager: "bundler",
subdependency_metadata: [{
production: production_dep_names.include?(dependency.name)
}]
)
lockfiles.each do |lockfile|
parsed_lockfile(lockfile).specs.each do |dependency|
next if dependency.source.is_a?(::Bundler::Source::Path)

dependencies <<
Dependency.new(
name: dependency.name,
version: dependency_version(dependency.name, lockfile)&.to_s,
requirements: [],
package_manager: "bundler",
subdependency_metadata: [{
production: production_dep_names(lockfile).include?(dependency.name)
}]
)
end
end

dependencies
end

def parsed_gemfile
@parsed_gemfile ||=
def parsed_gemfile(gemfile)
@parsed_gemfiles ||= {}
@parsed_gemfiles[gemfile.name] ||=
SharedHelpers.in_a_temporary_repo_directory(base_directory,
repo_contents_path) do
write_temporary_dependency_files

NativeHelpers.run_bundler_subprocess(
bundler_version: bundler_version,
bundler_version: bundler_version(gemfile),
function: "parsed_gemfile",
args: {
gemfile_name: gemfile.name,
lockfile_name: lockfile&.name,
lockfile_name: lockfile(gemfile)&.name,
dir: Dir.pwd
}
)
Expand All @@ -153,6 +161,7 @@ def handle_eval_error(err)
raise Dependabot::DependencyFileNotEvaluatable, msg
end

# TODO: When do gemspecs have lockfiles?
def parsed_gemspec(file)
@parsed_gemspecs ||= {}
@parsed_gemspecs[file.name] ||=
Expand All @@ -161,11 +170,11 @@ def parsed_gemspec(file)
write_temporary_dependency_files

NativeHelpers.run_bundler_subprocess(
bundler_version: bundler_version,
bundler_version: bundler_version(nil),
function: "parsed_gemspec",
args: {
gemspec_name: file.name,
lockfile_name: lockfile&.name,
lockfile_name: "",
dir: Dir.pwd
}
)
Expand All @@ -192,7 +201,9 @@ def write_temporary_dependency_files
File.write(path, file.content)
end

File.write(lockfile.name, sanitized_lockfile_content) if lockfile
lockfiles.each do |lockfile|
File.write(lockfile.name, sanitized_lockfile_content(lockfile))
end
end

def check_required_files
Expand All @@ -202,15 +213,15 @@ def check_required_files
name.end_with?(".gemspec") && !name.include?("/")
end

return if gemfile
return if gemfiles.any?

raise "A gemspec or Gemfile must be provided!"
end

def dependency_version(dependency_name)
def dependency_version(dependency_name, lockfile)
return unless lockfile

spec = parsed_lockfile.specs.find { |s| s.name == dependency_name }
spec = parsed_lockfile(lockfile).specs.find { |s| s.name == dependency_name }

# Not all files in the Gemfile will appear in the Gemfile.lock. For
# instance, if a gem specifies `platform: [:windows]`, and the
Expand All @@ -225,9 +236,11 @@ def dependency_version(dependency_name)
spec.version
end

def gemfile
@gemfile ||= get_original_file("Gemfile") ||
get_original_file("gems.rb")
def gemfiles
@gemfiles ||= dependency_files.select do |file|
(file.name.start_with?("Gemfile") && !file.name.end_with?(".lock")) ||
file.name == "gems.rb"
end
end

def evaled_gemfiles
Expand All @@ -241,31 +254,43 @@ def evaled_gemfiles
reject { |f| f.name == "gems.locked" }
end

def lockfile
@lockfile ||= get_original_file("Gemfile.lock") ||
get_original_file("gems.locked")
def lockfiles
@lockfiles ||= dependency_files.select do |file|
(file.name.start_with?("Gemfile") && file.name.end_with?(".lock")) || file.name == "gems.locked"
end
end

def lockfile(gemfile)
return if gemfile.nil?

@matched_lockfiles ||= {}
@matched_lockfiles[gemfile.name] ||=
lockfiles.find do |lockfile|
lockfile.name == "#{gemfile.name}.lock" || (gemfile.name == "gems.rb" && lockfile.name == "gems.locked")
end
end

def parsed_lockfile
@parsed_lockfile ||=
::Bundler::LockfileParser.new(sanitized_lockfile_content)
def parsed_lockfile(lockfile)
@parsed_lockfiles ||= {}
@parsed_lockfiles[lockfile.name] ||=
::Bundler::LockfileParser.new(sanitized_lockfile_content(lockfile))
end

def production_dep_names
def production_dep_names(lockfile)
@production_dep_names ||=
(gemfile_dependencies + gemspec_dependencies).dependencies.
select { |dep| production?(dep) }.
flat_map { |dep| expanded_dependency_names(dep) }.
flat_map { |dep| expanded_dependency_names(lockfile, dep) }.
uniq
end

def expanded_dependency_names(dep)
spec = parsed_lockfile.specs.find { |s| s.name == dep.name }
def expanded_dependency_names(lockfile, dep)
spec = parsed_lockfile(lockfile).specs.find { |s| s.name == dep.name }
return [dep.name] unless spec

[
dep.name,
*spec.dependencies.flat_map { |d| expanded_dependency_names(d) }
*spec.dependencies.flat_map { |d| expanded_dependency_names(lockfile, d) }
]
end

Expand All @@ -282,7 +307,7 @@ def production?(dependency)
end

# TODO: Stop sanitizing the lockfile once we have bundler 2 installed
def sanitized_lockfile_content
def sanitized_lockfile_content(lockfile)
regex = FileUpdater::LockfileUpdater::LOCKFILE_ENDING
lockfile.content.gsub(regex, "")
end
Expand All @@ -300,8 +325,11 @@ def imported_ruby_files
reject { |f| f.name == "gems.rb" }
end

def bundler_version
@bundler_version ||= Helpers.bundler_version(lockfile)
def bundler_version(gemfile)
@bundler_versions ||= {}
return Helpers.bundler_version(nil) if gemfile.nil?

@bundler_versions[gemfile.name] ||= Helpers.bundler_version(lockfile(gemfile))
end
end
end
Expand Down