Skip to content

Commit

Permalink
FileParser: Supports multiple gemfiles
Browse files Browse the repository at this point in the history
  • Loading branch information
@davidwessman
davidwessman committed Mar 11, 2021
1 parent c6eae14 commit 935f18c
Show file tree
Hide file tree
Showing 2 changed files with 76 additions and 48 deletions.
122 changes: 74 additions & 48 deletions bundler/lib/dependabot/bundler/file_parser.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,23 +45,25 @@ def git_source?(dependencies)
def gemfile_dependencies
dependencies = DependencySet.new

return dependencies unless gemfile
return dependencies unless gemfiles.any?

[gemfile, *evaled_gemfiles].each do |file|
parsed_gemfile.each do |dep|
all_gemfiles = gemfiles + evaled_gemfiles

all_gemfiles.each do |gemfile|
parsed_gemfile(gemfile).each do |dep|
gemfile_declaration_finder =
GemfileDeclarationFinder.new(dependency: dep, gemfile: file)
GemfileDeclarationFinder.new(dependency: dep, gemfile: gemfile)
next unless gemfile_declaration_finder.gemfile_includes_dependency?

dependencies <<
Dependency.new(
name: dep.fetch("name"),
version: dependency_version(dep.fetch("name"))&.to_s,
version: dependency_version(dep.fetch("name"), lockfile(gemfile))&.to_s,
requirements: [{
requirement: gemfile_declaration_finder.enhanced_req_string,
groups: dep.fetch("groups").map(&:to_sym),
source: dep.fetch("source")&.transform_keys(&:to_sym),
file: file.name
file: gemfile.name
}],
package_manager: "bundler"
)
Expand All @@ -71,15 +73,18 @@ def gemfile_dependencies
dependencies
end

# TODO: How to find the lockfile matching the gemspecs?
def gemspec_dependencies
dependencies = DependencySet.new

fallback_lockfile = lockfiles.first

gemspecs.each do |gemspec|
parsed_gemspec(gemspec).each do |dependency|
dependencies <<
Dependency.new(
name: dependency.fetch("name"),
version: dependency_version(dependency.fetch("name"))&.to_s,
version: dependency_version(dependency.fetch("name"), fallback_lockfile)&.to_s,
requirements: [{
requirement: dependency.fetch("requirement").to_s,
groups: if dependency.fetch("type") == "runtime"
Expand All @@ -101,41 +106,44 @@ def gemspec_dependencies
def lockfile_dependencies
dependencies = DependencySet.new

return dependencies unless lockfile
return dependencies unless lockfiles.any?

# Create a DependencySet where each element has no requirement. Any
# requirements will be added when combining the DependencySet with
# other DependencySets.
parsed_lockfile.specs.each do |dependency|
next if dependency.source.is_a?(::Bundler::Source::Path)

dependencies <<
Dependency.new(
name: dependency.name,
version: dependency_version(dependency.name)&.to_s,
requirements: [],
package_manager: "bundler",
subdependency_metadata: [{
production: production_dep_names.include?(dependency.name)
}]
)
lockfiles.each do |lockfile|
parsed_lockfile(lockfile).specs.each do |dependency|
next if dependency.source.is_a?(::Bundler::Source::Path)

dependencies <<
Dependency.new(
name: dependency.name,
version: dependency_version(dependency.name, lockfile)&.to_s,
requirements: [],
package_manager: "bundler",
subdependency_metadata: [{
production: production_dep_names(lockfile).include?(dependency.name)
}]
)
end
end

dependencies
end

def parsed_gemfile
@parsed_gemfile ||=
def parsed_gemfile(gemfile)
@parsed_gemfiles ||= {}
@parsed_gemfiles[gemfile.name] ||=
SharedHelpers.in_a_temporary_repo_directory(base_directory,
repo_contents_path) do
write_temporary_dependency_files

NativeHelpers.run_bundler_subprocess(
bundler_version: bundler_version,
bundler_version: bundler_version(gemfile),
function: "parsed_gemfile",
args: {
gemfile_name: gemfile.name,
lockfile_name: lockfile&.name,
lockfile_name: lockfile(gemfile)&.name,
dir: Dir.pwd
}
)
Expand All @@ -153,6 +161,7 @@ def handle_eval_error(err)
raise Dependabot::DependencyFileNotEvaluatable, msg
end

# TODO: When do gemspecs have lockfiles?
def parsed_gemspec(file)
@parsed_gemspecs ||= {}
@parsed_gemspecs[file.name] ||=
Expand All @@ -161,11 +170,11 @@ def parsed_gemspec(file)
write_temporary_dependency_files

NativeHelpers.run_bundler_subprocess(
bundler_version: bundler_version,
bundler_version: bundler_version(nil),
function: "parsed_gemspec",
args: {
gemspec_name: file.name,
lockfile_name: lockfile&.name,
lockfile_name: '',
dir: Dir.pwd
}
)
Expand All @@ -192,7 +201,9 @@ def write_temporary_dependency_files
File.write(path, file.content)
end

File.write(lockfile.name, sanitized_lockfile_content) if lockfile
lockfiles.each do |lockfile|
File.write(lockfile.name, sanitized_lockfile_content(lockfile))
end
end

def check_required_files
Expand All @@ -202,15 +213,16 @@ def check_required_files
name.end_with?(".gemspec") && !name.include?("/")
end

return if gemfile
return if gemfiles.any?

raise "A gemspec or Gemfile must be provided!"
end

def dependency_version(dependency_name)
def dependency_version(dependency_name, lockfile)
return unless lockfile

spec = parsed_lockfile.specs.find { |s| s.name == dependency_name }

spec = parsed_lockfile(lockfile).specs.find { |s| s.name == dependency_name }

# Not all files in the Gemfile will appear in the Gemfile.lock. For
# instance, if a gem specifies `platform: [:windows]`, and the
Expand All @@ -225,9 +237,11 @@ def dependency_version(dependency_name)
spec.version
end

def gemfile
@gemfile ||= get_original_file("Gemfile") ||
get_original_file("gems.rb")
def gemfiles
@gemfiles ||= dependency_files.select do |file|
(file.name.start_with?("Gemfile") && !file.name.end_with?(".lock")) ||
file.name == "gems.rb"
end
end

def evaled_gemfiles
Expand All @@ -241,31 +255,41 @@ def evaled_gemfiles
reject { |f| f.name == "gems.locked" }
end

def lockfile
@lockfile ||= get_original_file("Gemfile.lock") ||
get_original_file("gems.locked")

def lockfiles
@lockfiles ||= dependency_files.select do |file|
(file.name.start_with?("Gemfile") && file.name.end_with?(".lock")) || file.name == "gems.locked"
end
end

def lockfile(gemfile)
return if gemfile.nil?
@matched_lockfiles ||= {}
@matched_lockfiles[gemfile.name] ||=
lockfiles.find { |lockfile| lockfile.name == "#{gemfile.name}.lock" || (gemfile.name == "gems.rb" && lockfile.name == "gems.locked")}
end

def parsed_lockfile
@parsed_lockfile ||=
::Bundler::LockfileParser.new(sanitized_lockfile_content)
def parsed_lockfile(lockfile)
@parsed_lockfiles ||= {}
@parsed_lockfiles[lockfile.name] ||=
::Bundler::LockfileParser.new(sanitized_lockfile_content(lockfile))
end

def production_dep_names
def production_dep_names(lockfile)
@production_dep_names ||=
(gemfile_dependencies + gemspec_dependencies).dependencies.
select { |dep| production?(dep) }.
flat_map { |dep| expanded_dependency_names(dep) }.
flat_map { |dep| expanded_dependency_names(lockfile, dep) }.
uniq
end

def expanded_dependency_names(dep)
spec = parsed_lockfile.specs.find { |s| s.name == dep.name }
def expanded_dependency_names(lockfile, dep)
spec = parsed_lockfile(lockfile).specs.find { |s| s.name == dep.name }
return [dep.name] unless spec

[
dep.name,
*spec.dependencies.flat_map { |d| expanded_dependency_names(d) }
*spec.dependencies.flat_map { |d| expanded_dependency_names(lockfile, d) }
]
end

Expand All @@ -282,7 +306,7 @@ def production?(dependency)
end

# TODO: Stop sanitizing the lockfile once we have bundler 2 installed
def sanitized_lockfile_content
def sanitized_lockfile_content(lockfile)
regex = FileUpdater::LockfileUpdater::LOCKFILE_ENDING
lockfile.content.gsub(regex, "")
end
Expand All @@ -300,8 +324,10 @@ def imported_ruby_files
reject { |f| f.name == "gems.rb" }
end

def bundler_version
@bundler_version ||= Helpers.bundler_version(lockfile)
def bundler_version(gemfile)
@bundler_versions ||= {}
return Helpers.bundler_version(nil) if gemfile.nil?
@bundler_versions[gemfile.name] ||= Helpers.bundler_version(lockfile(gemfile))
end
end
end
Expand Down
2 changes: 2 additions & 0 deletions bundler/spec/dependabot/bundler/file_parser_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,7 @@
it { is_expected.to be_a(Dependabot::Dependency) }
its(:requirements) { is_expected.to eq(expected_requirements) }
its(:version) do
byebug
is_expected.to eq("d31e445215b5af70c1604715d97dd953e868380e")
end
end
Expand Down Expand Up @@ -508,6 +509,7 @@
expect(dependencies.first.name).to eq("business")
expect(dependencies.first.version).
to eq("1378a2b0b446d991b7567efbc7eeeed2720e4d8f")

expect(dependencies.first.requirements).
to match_array(
[{
Expand Down

0 comments on commit 935f18c

Please sign in to comment.