From b8d83a06b2c9cb2957781cbc04391ad2e7e83028 Mon Sep 17 00:00:00 2001 From: Zach Margolis Date: Fri, 19 Mar 2021 11:46:27 -0700 Subject: [PATCH] Download web.yml or worker.yml if it exists in S3 **Why**: This lets us configure the worker and web hosts with separate secrets and configs if needed --- lib/deploy/activate.rb | 35 +++++++++++- spec/lib/deploy/activate_spec.rb | 92 ++++++++++++++++++++++++++++++-- 2 files changed, 121 insertions(+), 6 deletions(-) diff --git a/lib/deploy/activate.rb b/lib/deploy/activate.rb index da9c4b2f03e..989f8509829 100644 --- a/lib/deploy/activate.rb +++ b/lib/deploy/activate.rb @@ -28,6 +28,7 @@ def run s3_path: '/%s/idp/v1/application.yml', local_path: env_yaml_path, ) + download_web_or_worker_yml_if_exists deep_merge_s3_data_with_example_application_yml set_proper_file_permissions_for_application_yml @@ -103,6 +104,17 @@ def deep_merge_s3_data_with_example_application_yml File.open(result_yaml_path, 'w') { |file| file.puts YAML.dump(application_config) } end + def download_web_or_worker_yml_if_exists + return unless web_or_worker_yml + + app_secrets_s3.download_file( + s3_path: "/%s/idp/v1/#{web_or_worker_yml}", + local_path: web_or_worker_yaml_path, + ) + rescue Aws::S3::Errors::NoSuchKey => err + logger.warn("did not load #{web_or_worker_yml}, continuing") + end + def set_proper_file_permissions_for_application_yml FileUtils.chmod(0o640, [env_yaml_path, result_yaml_path]) end @@ -147,7 +159,14 @@ def root end def application_config - YAML.load_file(example_application_yaml_path).deep_merge(YAML.load_file(env_yaml_path)) + config = YAML.load_file(example_application_yaml_path). + deep_merge(YAML.load_file(env_yaml_path)) + + if File.exist?(web_or_worker_yaml_path) + config.deep_merge(YAML.load_file(web_or_worker_yaml_path)) + else + config + end end def example_application_yaml_path @@ -169,5 +188,19 @@ def geolocation_db_path def pwned_passwords_path File.join(root, 'pwned_passwords/pwned_passwords.txt') end + + def web_or_worker_yaml_path + File.join(root, "config/#{web_or_worker_yml}") + end + + # @return [String, nil] + def web_or_worker_yml + case Identity::Hostdata.instance_role + when 'idp' + 'web.yml' + when 'worker' + 'worker.yml' + end + end end end diff --git a/spec/lib/deploy/activate_spec.rb b/spec/lib/deploy/activate_spec.rb index 100b7c6a91c..854aa2e62ee 100644 --- a/spec/lib/deploy/activate_spec.rb +++ b/spec/lib/deploy/activate_spec.rb @@ -17,6 +17,7 @@ let(:logger) { Logger.new('/dev/null') } let(:s3_client) { Aws::S3::Client.new(stub_responses: true) } let(:set_up_files!) {} + let(:instance_role) { 'idp' } let(:result_yaml_path) { File.join(root, 'config', 'application.yml') } let(:env_yaml_path) { File.join(root, 'config', 'application_s3_env.yml') } @@ -35,6 +36,7 @@ context 'in a deployed production environment' do before do allow(Identity::Hostdata).to receive(:env).and_return('int') + allow(Identity::Hostdata).to receive(:instance_role).and_return(instance_role) stub_request(:get, 'http://169.254.169.254/2016-09-02/dynamic/instance-identity/document'). to_return(body: { @@ -42,11 +44,15 @@ 'accountId' => '12345', }.to_json) - s3_client.stub_responses( - :get_object, - { body: application_yml }, - { body: geolite_content }, - { body: pwned_passwords_content }, + s3_client.stub_responses(:get_object, proc do |context| + key = context.params[:key] + body = s3_contents[key] + if body.present? + { body: body } + else + raise Aws::S3::Errors::NoSuchKey.new(nil, nil) + end + end ) allow(s3_client).to receive(:get_object).and_call_original @@ -55,6 +61,14 @@ allow(subject).to receive(:setup_idp_config_symlinks) end + let(:s3_contents) do + { + 'int/idp/v1/application.yml' => application_yml, + 'common/GeoIP2-City.mmdb' => geolite_content, + 'common/pwned-passwords.txt' => pwned_passwords_content, + } + end + let(:application_yml) do <<~YAML production: @@ -89,6 +103,74 @@ expect(combined_application_yml['production']['lockout_period_in_minutes']).to eq('10') end + context 'on a web instance' do + let(:instance_role) { 'idp' } + + context 'when web.yml exists in s3' do + before do + s3_contents['int/idp/v1/web.yml'] = <<~YAML + web_yaml_value: 'true' + YAML + end + + it 'merges web.yml into application.yml' do + subject.run + + expect(File.exist?("#{root}/config/web.yml")).to eq(true) + + combined_application_yml = YAML.load_file(result_yaml_path) + expect(combined_application_yml['web_yaml_value']).to eq('true') + end + end + + context 'when web.yml does not exist in s3' do + it 'warns and leaves application.yml as-is' do + expect(logger).to receive(:warn).with(/web.yml/) + + expect { subject.run }.to_not raise_error + + expect(File.exist?("#{root}/config/web.yml")).to eq(false) + + combined_application_yml = YAML.load_file(result_yaml_path) + expect(combined_application_yml).to_not have_key('web_yaml_value') + end + end + end + + context 'on a worker instance' do + let(:instance_role) { 'worker' } + + context 'when worker.yml exists in s3' do + before do + s3_contents['int/idp/v1/worker.yml'] = <<~YAML + worker_yaml_value: 'true' + YAML + end + + it 'merges worker.yml into application.yml' do + subject.run + + expect(File.exist?("#{root}/config/worker.yml")).to eq(true) + + combined_application_yml = YAML.load_file(result_yaml_path) + expect(combined_application_yml['worker_yaml_value']).to eq('true') + end + end + + context 'when worker.yml does not exist in s3' do + it 'warns and leaves application.yml as-is' do + expect(logger).to receive(:warn).with(/worker.yml/) + + expect { subject.run }.to_not raise_error + + expect(File.exist?("#{root}/config/worker.yml")).to eq(false) + + combined_application_yml = YAML.load_file(result_yaml_path) + expect(combined_application_yml).to_not have_key('worker_yaml_value') + end + end + end + it 'sets the correct permissions on the YAML files' do subject.run