From 526d520e023eaca01cf6f6bc4a9b5100b7d9f4f3 Mon Sep 17 00:00:00 2001 From: Dan Buch Date: Sun, 29 Mar 2015 21:09:19 -0400 Subject: [PATCH] Fold together the APT source and package addons --- .gitignore | 1 + .rspec | 3 + .simplecov | 8 ++ .travis.yml | 16 +-- Gemfile | 2 + Gemfile.lock | 15 +++ lib/travis/build/addons.rb | 1 + lib/travis/build/addons/apt.rb | 149 ++++++++++++++++++++++++ lib/travis/build/addons/apt_packages.rb | 60 +--------- lib/travis/build/addons/apt_sources.rb | 61 +--------- spec/build/addons/apt_packages_spec.rb | 5 +- spec/build/addons/apt_sources_spec.rb | 4 +- spec/build/addons/apt_spec.rb | 71 +++++++++++ spec/spec_helper.rb | 1 + 14 files changed, 265 insertions(+), 132 deletions(-) create mode 100644 .rspec create mode 100644 .simplecov create mode 100644 lib/travis/build/addons/apt.rb create mode 100644 spec/build/addons/apt_spec.rb diff --git a/.gitignore b/.gitignore index 3d48b71741..75a6688259 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ play/*.sh examples/*.txt rspec.log .vagrant +coverage/ diff --git a/.rspec b/.rspec new file mode 100644 index 0000000000..0109937da5 --- /dev/null +++ b/.rspec @@ -0,0 +1,3 @@ +--require spec_helper +--require pry +--format documentation diff --git a/.simplecov b/.simplecov new file mode 100644 index 0000000000..83d5bbc1ef --- /dev/null +++ b/.simplecov @@ -0,0 +1,8 @@ +# vim:filetype=ruby +SimpleCov.start do + add_filter '/spec/' + add_filter '/script/' + add_filter '/examples/' + add_filter '/play/' + add_filter '/tmp/' +end if ENV['COVERAGE'] diff --git a/.travis.yml b/.travis.yml index d9c9fee3f1..9da4e37dc0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,10 @@ sudo: false env: global: - RUBYOPT='-W0' + - COVERAGE=1 + +script: bundle exec rspec spec -script: "bundle exec rspec spec" before_deploy: ruby script/build_s3_index_html.rb deploy: @@ -24,15 +26,3 @@ deploy: region: us-east-1 on: branch: master - -# addons: -# artifacts: -# debug: yep -# permissions: public-read -# paths: -# - examples -# bucket: travis-build-examples -# key: -# secure: bxFEFpuTK00a/ZKQJK9B3ausZweNSd5ZmaeKSk9ozSkaXtqIQEt56C3hlq62PEyltXnnZhirbtYjw9QIrgggMhupKQCkwXEB7l/zgq/y9BDjDNZK8PW2YsJUkLIM5K3hITbW2zgWLJX7zic9W0EHQdoSZ7NW4FlNg47FlyIIJss= -# secret: -# secure: XZWMXcs2XxJwyQw/uEQbXNADwfzP7R5dGRzSrM8xvA6ErgIEh+ZcMuT7p7iPPWyaGUO9+MOTqM9HqdBcUuigGH4Vqn1e4MQ/LQsxAQWyuostzjuuo7k2q8OgOIXwcvg1+4LaVA4b4GQftw3lqkQ09Xa7yB/X8Y2/IlzpbbpB2YY= diff --git a/Gemfile b/Gemfile index 75fb7cdd6a..1db5b00053 100644 --- a/Gemfile +++ b/Gemfile @@ -20,6 +20,8 @@ end group :test do gem 'mocha', '~> 0.10.0' + gem 'pry' gem 'rspec', '~> 3.0' + gem 'simplecov', require: false gem 'sinatra-contrib' end diff --git a/Gemfile.lock b/Gemfile.lock index 3fb5308361..870bb81363 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -27,7 +27,9 @@ GEM celluloid (0.15.2) timers (~> 1.1.0) coder (0.4.0) + coderay (1.1.0) diff-lcs (1.2.5) + docile (1.1.5) faraday (0.9.0) multipart-post (>= 1.2, < 3) ffi (1.9.3) @@ -40,6 +42,7 @@ GEM rb-fsevent (>= 0.9.3) rb-inotify (>= 0.9) metaclass (0.0.4) + method_source (0.8.2) metriks (0.9.9.6) atomic (~> 1.0) avl_tree (~> 1.1.2) @@ -49,6 +52,10 @@ GEM metaclass (~> 0.0.1) multi_json (1.10.1) multipart-post (2.0.0) + pry (0.10.1) + coderay (~> 1.1.0) + method_source (~> 0.8.1) + slop (~> 3.4) puma (2.8.2) rack (>= 1.1, < 2.0) rack (1.5.2) @@ -79,6 +86,11 @@ GEM faraday (>= 0.7.6) hashie (>= 1.1.0) uuidtools + simplecov (0.9.1) + docile (~> 1.1.0) + multi_json (~> 1.0) + simplecov-html (~> 0.8.0) + simplecov-html (0.8.0) sinatra (1.4.5) rack (~> 1.4) rack-protection (~> 1.4) @@ -90,6 +102,7 @@ GEM rack-test sinatra (~> 1.4.0) tilt (~> 1.3) + slop (3.6.0) thread_safe (0.3.4) tilt (1.4.1) timers (1.1.0) @@ -107,11 +120,13 @@ DEPENDENCIES metriks (= 0.9.9.6) metriks-librato_metrics! mocha (~> 0.10.0) + pry puma rack-ssl (~> 1.4) rerun rspec (~> 3.0) sentry-raven + simplecov sinatra (~> 1.4) sinatra-contrib travis-support! diff --git a/lib/travis/build/addons.rb b/lib/travis/build/addons.rb index 07960b7601..411c0d1686 100644 --- a/lib/travis/build/addons.rb +++ b/lib/travis/build/addons.rb @@ -1,4 +1,5 @@ require 'active_support/core_ext/string/inflections.rb' +require 'travis/build/addons/apt' require 'travis/build/addons/apt_packages' require 'travis/build/addons/apt_sources' require 'travis/build/addons/artifacts' diff --git a/lib/travis/build/addons/apt.rb b/lib/travis/build/addons/apt.rb new file mode 100644 index 0000000000..dff1b6ff1c --- /dev/null +++ b/lib/travis/build/addons/apt.rb @@ -0,0 +1,149 @@ +require 'travis/build/addons/base' + +module Travis + module Build + class Addons + class Apt < Base + SUPER_USER_SAFE = true + + class << self + def package_whitelist + @package_whitelist ||= load_package_whitelist + end + + def source_whitelist + @source_whitelist ||= load_source_whitelist + end + + private + + def load_package_whitelist + require 'faraday' + response = fetch_package_whitelist + response.split.map(&:strip).sort.uniq + rescue => e + warn e + [] + end + + def load_source_whitelist + require 'faraday' + response = fetch_source_whitelist + entries = JSON.parse(response) + Hash[entries.reject { |e| !e.key?('alias') }.map { |e| [e.fetch('alias'), e] }] + rescue => e + warn e + {} + end + + def fetch_package_whitelist + Faraday.get(package_whitelist_url).body.to_s + end + + def fetch_source_whitelist + Faraday.get(source_whitelist_url).body.to_s + end + + def package_whitelist_url + ENV['TRAVIS_BUILD_APT_PACKAGE_WHITELIST'] || ENV['TRAVIS_BUILD_APT_WHITELIST'] + end + + def source_whitelist_url + ENV['TRAVIS_BUILD_APT_SOURCE_WHITELIST'] + end + end + + def after_prepare? + !config_sources.empty? && !config_packages.empty? + end + + def after_prepare + sh.fold('apt') do + sh.fold('apt.sources') { add_apt_sources } unless config_sources.empty? + sh.fold('apt.packages') { add_apt_packages } unless config_packages.empty? + end + end + + private + + def add_apt_sources + sh.echo "Adding APT Sources (BETA)", ansi: :yellow + + whitelisted = [] + disallowed = [] + + config_sources.each do |source_alias| + source = source_whitelist[source_alias] + whitelisted << source.clone if source && source['sourceline'] + disallowed << source_alias if source.nil? + end + + unless disallowed.empty? + sh.echo "Disallowing sources: #{disallowed.join(', ')}", ansi: :red + sh.echo 'If you require these sources, please review the source ' \ + 'approval process at: ' \ + 'https://github.com/travis-ci/apt-source-whitelist#source-approval-process' + end + + unless whitelisted.empty? + sh.export 'DEBIAN_FRONTEND', 'noninteractive', echo: true + whitelisted.each do |source| + sh.cmd "curl -sSL #{source['key_url'].untaint.inspect} | sudo -E apt-key add -", echo: true, assert: true, timing: true if source['key_url'] + sh.cmd "sudo -E apt-add-repository -y #{source['sourceline'].untaint.inspect}", echo: true, assert: true, timing: true + end + sh.cmd "sudo -E apt-get -yq update &>> ~/apt-get-update.log", echo: true, timing: true + end + end + + def add_apt_packages + sh.echo "Installing APT Packages (BETA)", ansi: :yellow + + whitelisted = [] + disallowed = [] + + config_packages.each do |package| + if package_whitelist.include?(package) + whitelisted << package + else + disallowed << package + end + end + + unless disallowed.empty? + sh.echo "Disallowing packages: #{disallowed.join(', ')}", ansi: :red + sh.echo 'If you require these packages, please review the package ' \ + 'approval process at: ' \ + 'https://github.com/travis-ci/apt-package-whitelist#package-approval-process' + end + + unless whitelisted.empty? + sh.export 'DEBIAN_FRONTEND', 'noninteractive', echo: true + sh.cmd "sudo -E apt-get -yq update &>> ~/apt-get-update.log", echo: true, timing: true + sh.cmd 'sudo -E apt-get -yq --no-install-suggests --no-install-recommends ' \ + "install #{whitelisted.join(' ')}", echo: true, timing: true + end + end + + def config + @config ||= Hash(super) + end + + def config_sources + Array(config[:sources]) + end + + def config_packages + Array(config[:packages]) + end + + def package_whitelist + ::Travis::Build::Addons::Apt.package_whitelist + end + + def source_whitelist + ::Travis::Build::Addons::Apt.source_whitelist + end + end + end + end +end diff --git a/lib/travis/build/addons/apt_packages.rb b/lib/travis/build/addons/apt_packages.rb index 17213b3524..59e87a3043 100644 --- a/lib/travis/build/addons/apt_packages.rb +++ b/lib/travis/build/addons/apt_packages.rb @@ -1,68 +1,16 @@ -require 'travis/build/addons/base' +require 'travis/build/addons/apt' module Travis module Build class Addons - class AptPackages < Base + class AptPackages SUPER_USER_SAFE = true class << self - def whitelist - @whitelist ||= load_whitelist - end - - private - - def load_whitelist - require 'faraday' - response = Faraday.get(ENV['TRAVIS_BUILD_APT_WHITELIST']) - response.body.to_s.split.map(&:strip).sort.uniq - rescue => e - warn e - [] + def new(script, sh, data, config) + ::Travis::Build::Addons::Apt.new(script, sh, data, { packages: config }) end end - - def after_prepare - sh.fold 'apt_packages' do - sh.echo "Installing APT Packages (BETA)", ansi: :yellow - - whitelisted = [] - disallowed = [] - - config.each do |package| - if whitelist.include?(package) - whitelisted << package - else - disallowed << package - end - end - - unless disallowed.empty? - sh.echo "Disallowing packages: #{disallowed.join(', ')}", ansi: :red - sh.echo 'If you require these packages, please review the package ' \ - 'approval process at: ' \ - 'https://github.com/travis-ci/apt-package-whitelist#package-approval-process' - end - - unless whitelisted.empty? - sh.export 'DEBIAN_FRONTEND', 'noninteractive', echo: true - sh.cmd "sudo -E apt-get -yq update &>> ~/apt-get-update.log", echo: true, timing: true - sh.cmd 'sudo -E apt-get -yq --no-install-suggests --no-install-recommends ' \ - "install #{whitelisted.join(' ')}", echo: true, timing: true - end - end - end - - private - - def config - Array(super) - end - - def whitelist - ::Travis::Build::Addons::AptPackages.whitelist - end end end end diff --git a/lib/travis/build/addons/apt_sources.rb b/lib/travis/build/addons/apt_sources.rb index 7be0e223e1..e49e035696 100644 --- a/lib/travis/build/addons/apt_sources.rb +++ b/lib/travis/build/addons/apt_sources.rb @@ -1,69 +1,16 @@ -require 'travis/build/addons/base' +require 'travis/build/addons/apt' module Travis module Build class Addons - class AptSources < Base + class AptSources SUPER_USER_SAFE = true class << self - def whitelist - @whitelist ||= load_whitelist - end - - private - - def load_whitelist - require 'faraday' - response = Faraday.get(ENV['TRAVIS_BUILD_APT_SOURCE_WHITELIST']) - entries = JSON.parse(response.body.to_s) - Hash[entries.reject { |e| !e.key?('alias') }.map { |e| [e.fetch('alias'), e] }] - rescue => e - warn e - {} + def new(script, sh, data, config) + ::Travis::Build::Addons::Apt.new(script, sh, data, { sources: config }) end end - - def after_prepare - sh.fold 'apt_sources' do - sh.echo "Adding APT Sources (BETA)", ansi: :yellow - - whitelisted = [] - disallowed = [] - - config.each do |source_alias| - source = whitelist[source_alias] - whitelisted << source.clone if source && source['sourceline'] - disallowed << source_alias if source.nil? - end - - unless disallowed.empty? - sh.echo "Disallowing sources: #{disallowed.join(', ')}", ansi: :red - sh.echo 'If you require these sources, please review the source ' \ - 'approval process at: ' \ - 'https://github.com/travis-ci/apt-source-whitelist#source-approval-process' - end - - unless whitelisted.empty? - sh.export 'DEBIAN_FRONTEND', 'noninteractive', echo: true - whitelisted.each do |source| - sh.cmd "curl -sSL #{source['key_url'].untaint.inspect} | sudo -E apt-key add -", echo: true, assert: true, timing: true if source['key_url'] - sh.cmd "sudo -E apt-add-repository -y #{source['sourceline'].untaint.inspect}", echo: true, assert: true, timing: true - end - sh.cmd "sudo -E apt-get -yq update &>> ~/apt-get-update.log", echo: true, timing: true - end - end - end - - private - - def config - Array(super) - end - - def whitelist - ::Travis::Build::Addons::AptSources.whitelist - end end end end diff --git a/spec/build/addons/apt_packages_spec.rb b/spec/build/addons/apt_packages_spec.rb index e745d01198..c5b81b2fe3 100644 --- a/spec/build/addons/apt_packages_spec.rb +++ b/spec/build/addons/apt_packages_spec.rb @@ -1,16 +1,15 @@ require 'ostruct' -require 'spec_helper' describe Travis::Build::Addons::AptPackages, :sexp do let(:script) { stub('script') } let(:data) { payload_for(:push, :ruby, config: { addons: { apt_packages: config } }) } let(:sh) { Travis::Shell::Builder.new } let(:addon) { described_class.new(script, sh, Travis::Build::Data.new(data), config) } - let(:whitelist) { ['curl', 'git'] } + let(:package_whitelist) { ['curl', 'git'] } subject { sh.to_sexp } before do - addon.stubs(:whitelist).returns(whitelist) + addon.stubs(:package_whitelist).returns(package_whitelist) addon.after_prepare end diff --git a/spec/build/addons/apt_sources_spec.rb b/spec/build/addons/apt_sources_spec.rb index 5707a1f7a5..c7a1153a73 100644 --- a/spec/build/addons/apt_sources_spec.rb +++ b/spec/build/addons/apt_sources_spec.rb @@ -1,5 +1,3 @@ -require 'spec_helper' - describe Travis::Build::Addons::AptSources, :sexp do let(:script) { stub('script') } let(:data) { payload_for(:push, :ruby, config: { addons: { apt_sources: config } }) } @@ -28,7 +26,7 @@ subject { sh.to_sexp } before do - addon.stubs(:whitelist).returns(whitelist) + addon.stubs(:source_whitelist).returns(whitelist) addon.after_prepare end diff --git a/spec/build/addons/apt_spec.rb b/spec/build/addons/apt_spec.rb new file mode 100644 index 0000000000..376e9eb792 --- /dev/null +++ b/spec/build/addons/apt_spec.rb @@ -0,0 +1,71 @@ +require 'faraday' +require 'json' + +describe Travis::Build::Addons::Apt, :sexp do + let(:script) { stub('script') } + let(:data) { payload_for(:push, :ruby, config: { addons: { apt: config } }) } + let(:sh) { Travis::Shell::Builder.new } + let(:addon) { described_class.new(script, sh, Travis::Build::Data.new(data), config) } + let(:config) { {} } + let(:source_whitelist) { [{ alias: 'testing', sourceline: 'deb http://example.com/deb repo main' }] } + let(:package_whitelist) { %w(git curl) } + subject { sh.to_sexp } + + before :all do + Faraday.default_adapter = :test + end + + before do + described_class.instance_variable_set(:@package_whitelist, nil) + described_class.instance_variable_set(:@source_whitelist, nil) + addon.after_prepare + end + + context 'when the package whitelist is provided' do + before do + described_class.stubs(:fetch_package_whitelist).returns(package_whitelist.join("\n")) + end + + it 'exposes a package whitelist' do + expect(described_class.package_whitelist).to_not be_empty + end + + it 'instances delegate package whitelist to class' do + expect(described_class.package_whitelist.object_id).to eql(addon.send(:package_whitelist).object_id) + end + end + + context 'when the source whitelist is provided' do + before do + described_class.stubs(:fetch_source_whitelist).returns(JSON.dump(source_whitelist)) + end + + it 'exposes a source whitelist' do + expect(described_class.source_whitelist).to_not be_empty + end + + it 'instances delegate source whitelist to class' do + expect(described_class.source_whitelist.object_id).to eql(addon.send(:source_whitelist).object_id) + end + end + + context 'when the package whitelist cannot be fetched' do + before do + described_class.stubs(:fetch_package_whitelist).raises(StandardError) + end + + it 'defaults package whitelist to empty array' do + expect(described_class.package_whitelist).to eql([]) + end + end + + context 'when the source whitelist cannot be fetched' do + before do + described_class.stubs(:fetch_source_whitelist).raises(StandardError) + end + + it 'defaults source whitelist to empty hash' do + expect(described_class.source_whitelist).to eql({}) + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index b814e7c7d8..97dd082bcb 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,3 +1,4 @@ +require 'simplecov' require 'fileutils' require 'sinatra/test_helpers' require 'travis/build'