From 65fcaf083c8a6a9eda0d49019481f268f73154b3 Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Thu, 6 Jul 2023 08:28:27 -0700 Subject: [PATCH] Endpoint URL Configuration --- gems/aws-sdk-core/CHANGELOG.md | 2 + .../lib/aws-sdk-core/ini_parser.rb | 2 +- .../aws-sdk-core/plugins/regional_endpoint.rb | 71 +++++++-- .../lib/aws-sdk-core/shared_config.rb | 23 ++- .../lib/seahorse/client/configuration.rb | 4 - gems/aws-sdk-core/spec/aws/ini_parser_spec.rb | 13 +- .../aws/plugins/regional_endpoint_spec.rb | 140 +++++++++++++++--- .../spec/aws/shared_config_spec.rb | 32 ++++ .../fixtures/credentials/mock_shared_config | 16 +- 9 files changed, 265 insertions(+), 38 deletions(-) diff --git a/gems/aws-sdk-core/CHANGELOG.md b/gems/aws-sdk-core/CHANGELOG.md index a582ffb6979..6db96542298 100644 --- a/gems/aws-sdk-core/CHANGELOG.md +++ b/gems/aws-sdk-core/CHANGELOG.md @@ -1,6 +1,8 @@ Unreleased Changes ------------------ +* Feature - Add support for configuring the endpoint URL in the shared configuration file or via an environment variable for a specific AWS service or all AWS services. + 3.176.1 (2023-06-29) ------------------ diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/ini_parser.rb b/gems/aws-sdk-core/lib/aws-sdk-core/ini_parser.rb index 4fcadb3200c..268a2980e63 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/ini_parser.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/ini_parser.rb @@ -17,7 +17,7 @@ def ini_parse(raw) current_profile = named_profile[1] if named_profile elsif current_profile unless line.nil? - item = line.match(/^(.+?)\s*=\s*(.+?)\s*$/) + item = line.match(/^(.+?)\s*=\s*([^\s].*?)\s*$/) prefix = line.match(/^(.+?)\s*=\s*$/) end if item && item[1].match(/^\s+/) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb index 729578e03ac..f7947929c49 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb @@ -47,14 +47,26 @@ class RegionalEndpoint < Seahorse::Client::Plugin # Legacy endpoints must continue to be generated at client time. option(:regional_endpoint, false) - # NOTE: All of the defaults block code is effectively deprecated. - # Because old services can depend on this new core version, we must - # retain it. + option(:ignore_configured_endpoint_urls, default: nil, doc_type: 'Boolean', docstring: <<-DOCS) +Setting to true disables use of endpoint URLs provided via environment +variables and the shared configuration file. +DOCS + + # NOTE: with Endpoints 2.0, some of this logic is deprecated + # but because new old service gems may depend on new core versions + # we must preserve that behavior. + # Additional behavior controls the setting of the custom SDK::Endpoint + # parameter. + # When the `regional_endpoint` config is set to true - this indicates to + # Endpoints2.0 that a custom endpoint has NOT been configured by the user. option(:endpoint, doc_type: String, docstring: <<-DOCS) do |cfg| The client endpoint is normally constructed from the `:region` option. You should only configure an `:endpoint` when connecting to test or custom endpoints. This should be a valid HTTP(S) URI. DOCS + + config_endpoint = resolve_custom_config_endpoint(cfg) + endpoint_prefix = cfg.api.metadata['endpointPrefix'] if cfg.region && endpoint_prefix if cfg.respond_to?(:sts_regional_endpoints) @@ -66,6 +78,7 @@ class RegionalEndpoint < Seahorse::Client::Plugin raise Errors::InvalidRegionError end + # handle deprecated pseudo-regions region = cfg.region new_region = region.gsub('fips-', '').gsub('-fips', '') if region != new_region @@ -75,16 +88,24 @@ class RegionalEndpoint < Seahorse::Client::Plugin cfg.override_config(:region, new_region) end - Aws::Partitions::EndpointProvider.resolve( - cfg.region, - endpoint_prefix, - sts_regional, - { - dualstack: cfg.use_dualstack_endpoint, - fips: cfg.use_fips_endpoint - } - ) + unless config_endpoint + # set regional_endpoint flag - this indicates to Endpoints 2.0 that a custom endpoint has NOT been configured by the user + cfg.override_config(:regional_endpoint, true) + + # set a default endpoint in config + config_endpoint = Aws::Partitions::EndpointProvider.resolve( + cfg.region, + endpoint_prefix, + sts_regional, + { + dualstack: cfg.use_dualstack_endpoint, + fips: cfg.use_fips_endpoint + } + ) + end end + + config_endpoint end def after_initialize(client) @@ -117,6 +138,32 @@ def resolve_use_fips_endpoint(cfg) value ||= Aws.shared_config.use_fips_endpoint(profile: cfg.profile) Aws::Util.str_2_bool(value) || false end + + # get a custom configured endpoint from ENV or configuration + def resolve_custom_config_endpoint(cfg) + config_endpoint = nil + ignore_configured_endpoints = cfg.ignore_configured_endpoint_urls + if ignore_configured_endpoints.nil? + ignore_configured_endpoints = ENV['AWS_IGNORE_CONFIGURED_ENDPOINT_URLS'] + ignore_configured_endpoints ||= Aws.shared_config.ignore_configured_endpoint_urls(profile: cfg.profile) + ignore_configured_endpoints = Aws::Util.str_2_bool(ignore_configured_endpoints&.downcase) || false + end + unless ignore_configured_endpoints + service_id = cfg.api.metadata['serviceId'] || cfg.api.metadata['endpointPrefix'] + env_service_id = service_id.gsub(" ", "_").upcase + if (config_endpoint = ENV["AWS_ENDPOINT_URL_#{env_service_id}"]) + cfg.logger&.debug( + "Endpoint configured from ENV['AWS_ENDPOINT_URL_#{env_service_id}']: #{config_endpoint}\n") + elsif (config_endpoint = ENV['AWS_ENDPOINT_URL']) + cfg.logger&.debug( + "Endpoint configured from ENV['AWS_ENDPOINT_URL']: #{config_endpoint}\n") + elsif (config_endpoint = Aws.shared_config.configured_endpoint(profile: cfg.profile, service_id: service_id)) + cfg.logger&.debug( + "Endpoint configured from shared config(profile: #{cfg.profile}): #{config_endpoint}\n") + end + end + return config_endpoint + end end end end diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/shared_config.rb b/gems/aws-sdk-core/lib/aws-sdk-core/shared_config.rb index 6b23a1b73f6..b58c3111b38 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/shared_config.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/shared_config.rb @@ -167,6 +167,26 @@ def sso_token_from_config(opts = {}) token end + # Source a custom configured endpoint from the shared configuration file + # + # @param [Hash] options + # @option options [String] :profile + # @option options [String] :service_id + def configured_endpoint(opts = {}) + # services section is only allowed in the shared config file (not credentials) + profile = opts[:profile] || @profile_name + service_id = opts[:service_id]&.gsub(" ", "_")&.downcase + if @parsed_config && (prof_config = @parsed_config[profile]) + services_section_name = prof_config['services'] + if (services_config = @parsed_config["services #{services_section_name}"]) && + (service_config = services_config[service_id]) + return service_config['endpoint_url'] if service_config['endpoint_url'] + end + return prof_config['endpoint_url'] + end + nil + end + # Add an accessor method (similar to attr_reader) to return a configuration value # Uses the get_config_value below to control where # values are loaded from @@ -198,7 +218,8 @@ def self.config_reader(*attrs) :s3_us_east_1_regional_endpoint, :s3_disable_multiregion_access_points, :defaults_mode, - :sdk_ua_app_id + :sdk_ua_app_id, + :ignore_configured_endpoint_urls ) private diff --git a/gems/aws-sdk-core/lib/seahorse/client/configuration.rb b/gems/aws-sdk-core/lib/seahorse/client/configuration.rb index 19cc73d13b0..683a053d26d 100644 --- a/gems/aws-sdk-core/lib/seahorse/client/configuration.rb +++ b/gems/aws-sdk-core/lib/seahorse/client/configuration.rb @@ -204,10 +204,6 @@ def override_config(k, v) def value_at(opt_name) value = @struct[opt_name] if value.is_a?(Defaults) - # Legacy endpoints must continue to exist. - if opt_name == :endpoint && @struct.members.include?(:regional_endpoint) - @struct[:regional_endpoint] = true - end resolve_defaults(opt_name, value) else value diff --git a/gems/aws-sdk-core/spec/aws/ini_parser_spec.rb b/gems/aws-sdk-core/spec/aws/ini_parser_spec.rb index 5c3b3d72e78..f94e5c0da4e 100644 --- a/gems/aws-sdk-core/spec/aws/ini_parser_spec.rb +++ b/gems/aws-sdk-core/spec/aws/ini_parser_spec.rb @@ -29,6 +29,11 @@ module Aws [sso-session dev] sso_region = us-east-1 + +[services test-services] +s3 = + endpoint_url = https://localhost:8000 + FILE } @@ -51,6 +56,12 @@ module Aws it 'can parse sso-session sections' do parsed = IniParser.ini_parse(mock_config) - expect(parsed['sso-session dev']['sso_region']).to eq('us-east-1') end + expect(parsed['sso-session dev']['sso_region']).to eq('us-east-1') + end + + it 'can parse services sections' do + parsed = IniParser.ini_parse(mock_config) + expect(parsed['services test-services']['s3']['endpoint_url']).to eq('https://localhost:8000') + end end end diff --git a/gems/aws-sdk-core/spec/aws/plugins/regional_endpoint_spec.rb b/gems/aws-sdk-core/spec/aws/plugins/regional_endpoint_spec.rb index 054feea964a..b5cfcdde103 100644 --- a/gems/aws-sdk-core/spec/aws/plugins/regional_endpoint_spec.rb +++ b/gems/aws-sdk-core/spec/aws/plugins/regional_endpoint_spec.rb @@ -6,14 +6,10 @@ module Aws module Plugins describe RegionalEndpoint do RegionalEndpointClient = ApiHelper.sample_service.const_get(:Client) - - let(:env) { {} } - + let(:client_class) { RegionalEndpointClient } - before do - stub_const('ENV', env) - end + let(:region) { 'REGION' } describe 'region option' do it 'adds a :region configuration option' do @@ -22,35 +18,35 @@ module Plugins end it 'defaults to ENV["AWS_DEFAULT_REGION"]' do - env['AWS_DEFAULT_REGION'] = 'env-region' + ENV['AWS_DEFAULT_REGION'] = 'env-region' expect(client_class.new.config.region).to eq('env-region') end it 'defaults to ENV["AWS_REGION"]' do - env['AWS_REGION'] = 'env-fallback1' + ENV['AWS_REGION'] = 'env-fallback1' expect(client_class.new.config.region).to eq('env-fallback1') end it 'falls back to ENV["AMAZON_REGION"]' do - env['AMAZON_REGION'] = 'region-fallback2' + ENV['AMAZON_REGION'] = 'region-fallback2' expect(client_class.new.config.region).to eq('region-fallback2') end it 'prefers AWS_REGION to AMAZON_REGION or AWS_DEFAULT_REGION' do - env['AWS_REGION'] = 'aws-region' - env['AMAZON_REGION'] = 'amazon-region' - env['AWS_DEFAULT_REGION'] = 'aws-default-region' + ENV['AWS_REGION'] = 'aws-region' + ENV['AMAZON_REGION'] = 'amazon-region' + ENV['AWS_DEFAULT_REGION'] = 'aws-default-region' expect(client_class.new.config.region).to eq('aws-region') end it 'prefers AWS_REGION to AMAZON_REGION' do - env['AWS_REGION'] = 'aws-region' - env['AMAZON_REGION'] = 'amazon-region' + ENV['AWS_REGION'] = 'aws-region' + ENV['AMAZON_REGION'] = 'amazon-region' expect(client_class.new.config.region).to eq('aws-region') end it 'can be set directly, overriding the ENV["AWS_REGION"]' do - env['AWS_REGION'] = 'env-region' + ENV['AWS_REGION'] = 'env-region' expect(client_class.new(region: 'cfg').config.region).to eq('cfg') end @@ -90,11 +86,119 @@ module Plugins end describe 'endpoint option' do - it 'defaults the endpoint to PREFIX.REGION.amazonaws.com' do + it 'preserves legacy pre-endpoints2.0 behavior and sets the endpoint and regional_endpoint' do prefix = client_class.api.metadata['endpointPrefix'] - client = client_class.new(region: 'REGION') + client = client_class.new(region: region) expect(client.config.endpoint.to_s) - .to eq("https://#{prefix}.REGION.amazonaws.com") + .to eq("https://#{prefix}.#{region}.amazonaws.com") + expect(client.config.regional_endpoint).to be_truthy + end + + context 'ENV[AWS_IGNORE_CONFIGURED_ENDPOINT_URLS] set' do + before { ENV['AWS_IGNORE_CONFIGURED_ENDPOINT_URLS'] = 'True' } + + it 'ignores ENV[AWS_ENDPOINT_URL]' do + ENV['AWS_ENDPOINT_URL'] = 'custom-env-url' + expect(client_class.new(region: region).config.endpoint.to_s) + .not_to eq('custom-env-url') + end + + it 'ignores endpoint_url in shared config' do + expect(Aws.shared_config).not_to receive(:configured_endpoint) + client_class.new(region: region) + end + + it 'uses client configured endpoint' do + expect(client_class.new( + region: region, endpoint: 'https://custom-client-endpoint' + ).config.endpoint.to_s).to eq('https://custom-client-endpoint') + end + end + + context 'Shared config ignore_configured_endpoint_urls set' do + before do + allow_any_instance_of(Aws::SharedConfig) + .to receive(:ignore_configured_endpoint_urls).and_return('true') + end + + it 'ignores ENV[AWS_ENDPOINT_URL]' do + ENV['AWS_ENDPOINT_URL'] = 'custom-env-url' + expect(client_class.new(region: region).config.endpoint.to_s) + .not_to eq('custom-env-url') + end + + it 'ignores endpoint_url in shared config' do + expect(Aws.shared_config).not_to receive(:configured_endpoint) + client_class.new(region: region) + end + + it 'uses client configured endpoint' do + expect(client_class.new( + region: region, endpoint: 'https://custom-client-endpoint' + ).config.endpoint.to_s).to eq('https://custom-client-endpoint') + end + end + + context 'client ignore_configured_endpoint_urls set' do + let(:cfg) do + { + region: region, + ignore_configured_endpoint_urls: true + } + end + + it 'ignores ENV[AWS_ENDPOINT_URL]' do + ENV['AWS_ENDPOINT_URL'] = 'custom-env-url' + expect(client_class.new(cfg).config.endpoint.to_s) + .not_to eq('custom-env-url') + end + + it 'ignores endpoint_url in shared config' do + expect(Aws.shared_config).not_to receive(:configured_endpoint) + client_class.new(cfg) + end + + it 'ignores ENV[AWS_ENDPOINT_URL] even when Shared config ignore_configured_endpoint_urls set' do + allow_any_instance_of(Aws::SharedConfig) + .to receive(:ignore_configured_endpoint_urls).and_return('true') + ENV['AWS_ENDPOINT_URL'] = 'custom-env-url' + expect(client_class.new(cfg).config.endpoint.to_s) + .not_to eq('custom-env-url') + end + end + + it 'uses configured_endpoint from shared config' do + allow(Aws.shared_config).to receive(:configured_endpoint) + .and_return('https://shared-config') + expect(client_class.new(region: region).config.endpoint.to_s) + .to eq('https://shared-config') + end + + it 'uses ENV[AWS_ENDPOINT_URL] over shared config' do + allow(Aws.shared_config).to receive(:configured_endpoint) + .and_return('https://shared-config') + ENV['AWS_ENDPOINT_URL'] = 'https://global-env' + expect(client_class.new(region: region).config.endpoint.to_s) + .to eq('https://global-env') + end + + it 'uses service specific ENV over ENV[AWS_ENDPOINT_URL]' do + allow(Aws.shared_config).to receive(:configured_endpoint) + .and_return('https://shared-config') + ENV['AWS_ENDPOINT_URL'] = 'https://global-env' + ENV['AWS_ENDPOINT_URL_SVC'] = 'https://service-env' + expect(client_class.new(region: region).config.endpoint.to_s) + .to eq('https://service-env') + end + + it 'uses client configured endpoint over all other configuration' do + allow(Aws.shared_config).to receive(:configured_endpoint) + .and_return('https://shared-config') + ENV['AWS_ENDPOINT_URL'] = 'https://global-env' + ENV['AWS_ENDPOINT_URL_SVC'] = 'https://service-env' + expect(client_class.new( + region: region, endpoint: 'https://client').config.endpoint.to_s) + .to eq('https://client') end end diff --git a/gems/aws-sdk-core/spec/aws/shared_config_spec.rb b/gems/aws-sdk-core/spec/aws/shared_config_spec.rb index ea2f8e48fed..1378841fa7c 100644 --- a/gems/aws-sdk-core/spec/aws/shared_config_spec.rb +++ b/gems/aws-sdk-core/spec/aws/shared_config_spec.rb @@ -349,5 +349,37 @@ module Aws expect(config.sdk_ua_app_id).to eq('peccy-service') end end + + context 'configured_endpoint' do + let(:config) do + SharedConfig.new( + config_path: mock_config_file, + config_enabled: true, + ) + end + it 'resolves configured_endpoint from global urls' do + expect(config.configured_endpoint(profile: 'global_endpoint_url')) + .to eq('https://play.min.io:9000') + end + + it 'resolves service specific endpoints over global urls' do + expect(config.configured_endpoint( + profile: 'service_specific_and_global_endpoint_url', + service_id: 's3')) + .to eq('https://play.min.io:9000') + + expect(config.configured_endpoint( + profile: 'service_specific_and_global_endpoint_url', + service_id: 'other_service')) + .to eq('http://localhost:1234') + end + + it 'transforms service_id' do + expect(config.configured_endpoint( + profile: 'service_specific_and_global_endpoint_url', + service_id: 'Elastic Beanstalk')) + .to eq('http://localhost:8000') + end + end end end diff --git a/gems/aws-sdk-core/spec/fixtures/credentials/mock_shared_config b/gems/aws-sdk-core/spec/fixtures/credentials/mock_shared_config index db635af89a3..b8d08037077 100644 --- a/gems/aws-sdk-core/spec/fixtures/credentials/mock_shared_config +++ b/gems/aws-sdk-core/spec/fixtures/credentials/mock_shared_config @@ -278,4 +278,18 @@ role_arn = arn:aws:iam:123456789012:role/role_b defaults_mode = standard [profile sdk_ua_app_id] -sdk_ua_app_id = peccy-service \ No newline at end of file +sdk_ua_app_id = peccy-service + +[profile global_endpoint_url] +endpoint_url = https://play.min.io:9000 + +[services dev_services] +s3 = + endpoint_url = https://play.min.io:9000 +elastic_beanstalk = + endpoint_url = http://localhost:8000 + +[profile service_specific_and_global_endpoint_url] +endpoint_url = http://localhost:1234 +services = dev_services +