Endpoint URL Configuration

gems/aws-sdk-core/CHANGELOG.md

@@ -1,6 +1,8 @@
 Unreleased Changes
 ------------------
 
+* Feature - Add support for configuring the endpoint URL in the shared configuration file or via an environment variable for a specific AWS service or all AWS services.
+
 3.176.1 (2023-06-29)
 ------------------
 

gems/aws-sdk-core/lib/aws-sdk-core/ini_parser.rb

@@ -17,7 +17,7 @@ def ini_parse(raw)
             current_profile = named_profile[1] if named_profile
           elsif current_profile
             unless line.nil?
-              item = line.match(/^(.+?)\s*=\s*(.+?)\s*$/)
+              item = line.match(/^(.+?)\s*=\s*([^\s].*?)\s*$/)
               prefix = line.match(/^(.+?)\s*=\s*$/)
             end
             if item && item[1].match(/^\s+/)

gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb

@@ -47,14 +47,26 @@ class RegionalEndpoint < Seahorse::Client::Plugin
       # Legacy endpoints must continue to be generated at client time.
       option(:regional_endpoint, false)
 
-      # NOTE: All of the defaults block code is effectively deprecated.
-      # Because old services can depend on this new core version, we must
-      # retain it.
+      option(:ignore_configured_endpoint_urls, default: nil, doc_type: 'Boolean', docstring: <<-DOCS)
+Setting to true disables use of endpoint URLs provided via environment 
+variables and the shared configuration file.
+DOCS
+
+      # NOTE: with Endpoints 2.0, some of this logic is deprecated
+      # but because new old service gems may depend on new core versions
+      # we must preserve that behavior.
+      # Additional behavior controls the setting of the custom SDK::Endpoint
+      # parameter.
+      # When the `regional_endpoint` config is set to true - this indicates to
+      # Endpoints2.0 that a custom endpoint has NOT been configured by the user.
       option(:endpoint, doc_type: String, docstring: <<-DOCS) do |cfg|
 The client endpoint is normally constructed from the `:region`
 option. You should only configure an `:endpoint` when connecting
 to test or custom endpoints. This should be a valid HTTP(S) URI.
         DOCS
+
+        config_endpoint = resolve_custom_config_endpoint(cfg)
+
         endpoint_prefix = cfg.api.metadata['endpointPrefix']
         if cfg.region && endpoint_prefix
           if cfg.respond_to?(:sts_regional_endpoints)
@@ -66,6 +78,7 @@ class RegionalEndpoint < Seahorse::Client::Plugin
             raise Errors::InvalidRegionError
           end
 
+          # handle deprecated pseudo-regions
           region = cfg.region
           new_region = region.gsub('fips-', '').gsub('-fips', '')
           if region != new_region
@@ -75,16 +88,24 @@ class RegionalEndpoint < Seahorse::Client::Plugin
             cfg.override_config(:region, new_region)
           end
 
-          Aws::Partitions::EndpointProvider.resolve(
-            cfg.region,
-            endpoint_prefix,
-            sts_regional,
-            {
-              dualstack: cfg.use_dualstack_endpoint,
-              fips: cfg.use_fips_endpoint
-            }
-          )
+          unless config_endpoint
+            # set regional_endpoint flag - this indicates to Endpoints 2.0 that a custom endpoint has NOT been configured by the user
+            cfg.override_config(:regional_endpoint, true)
+
+            # set a default endpoint in config
+            config_endpoint = Aws::Partitions::EndpointProvider.resolve(
+              cfg.region,
+              endpoint_prefix,
+              sts_regional,
+              {
+                dualstack: cfg.use_dualstack_endpoint,
+                fips: cfg.use_fips_endpoint
+              }
+            )
+          end
         end
+
+        config_endpoint
       end
 
       def after_initialize(client)
@@ -117,6 +138,32 @@ def resolve_use_fips_endpoint(cfg)
           value ||= Aws.shared_config.use_fips_endpoint(profile: cfg.profile)
           Aws::Util.str_2_bool(value) || false
         end
+
+        # get a custom configured endpoint from ENV or configuration
+        def resolve_custom_config_endpoint(cfg)
+          config_endpoint = nil
+          ignore_configured_endpoints = cfg.ignore_configured_endpoint_urls
+          if ignore_configured_endpoints.nil?
+            ignore_configured_endpoints = ENV['AWS_IGNORE_CONFIGURED_ENDPOINT_URLS']
+            ignore_configured_endpoints ||= Aws.shared_config.ignore_configured_endpoint_urls(profile: cfg.profile)
+            ignore_configured_endpoints = Aws::Util.str_2_bool(ignore_configured_endpoints&.downcase) || false
+          end
+          unless ignore_configured_endpoints
+            service_id = cfg.api.metadata['serviceId'] || cfg.api.metadata['endpointPrefix']
+            env_service_id = service_id.gsub(" ", "_").upcase
+            if (config_endpoint = ENV["AWS_ENDPOINT_URL_#{env_service_id}"])
+              cfg.logger&.debug(
+                "Endpoint configured from ENV['AWS_ENDPOINT_URL_#{env_service_id}']: #{config_endpoint}\n")
+            elsif (config_endpoint = ENV['AWS_ENDPOINT_URL'])
+              cfg.logger&.debug(
+                "Endpoint configured from ENV['AWS_ENDPOINT_URL']: #{config_endpoint}\n")
+            elsif (config_endpoint = Aws.shared_config.configured_endpoint(profile: cfg.profile, service_id: service_id))
+              cfg.logger&.debug(
+                "Endpoint configured from shared config(profile: #{cfg.profile}): #{config_endpoint}\n")
+            end
+          end
+          return config_endpoint
+        end
       end
     end
   end

gems/aws-sdk-core/lib/aws-sdk-core/shared_config.rb

@@ -167,6 +167,26 @@ def sso_token_from_config(opts = {})
       token
     end
 
+    # Source a custom configured endpoint from the shared configuration file
+    #
+    # @param [Hash] options
+    # @option options [String] :profile
+    # @option options [String] :service_id
+    def configured_endpoint(opts = {})
+      # services section is only allowed in the shared config file (not credentials)
+      profile = opts[:profile] || @profile_name
+      service_id = opts[:service_id]&.gsub(" ", "_")&.downcase
+      if @parsed_config && (prof_config = @parsed_config[profile])
+        services_section_name = prof_config['services']
+        if (services_config = @parsed_config["services #{services_section_name}"]) &&
+          (service_config = services_config[service_id])
+          return service_config['endpoint_url'] if service_config['endpoint_url']
+        end
+        return prof_config['endpoint_url']
+      end
+      nil
+    end
+
     # Add an accessor method (similar to attr_reader) to return a configuration value
     # Uses the get_config_value below to control where
     # values are loaded from
@@ -198,7 +218,8 @@ def self.config_reader(*attrs)
       :s3_us_east_1_regional_endpoint,
       :s3_disable_multiregion_access_points,
       :defaults_mode,
-      :sdk_ua_app_id
+      :sdk_ua_app_id,
+      :ignore_configured_endpoint_urls
     )
 
     private

gems/aws-sdk-core/lib/seahorse/client/configuration.rb

@@ -204,10 +204,6 @@ def override_config(k, v)
         def value_at(opt_name)
           value = @struct[opt_name]
           if value.is_a?(Defaults)
-            # Legacy endpoints must continue to exist.
-            if opt_name == :endpoint && @struct.members.include?(:regional_endpoint)
-              @struct[:regional_endpoint] = true
-            end
             resolve_defaults(opt_name, value)
           else
             value

gems/aws-sdk-core/spec/aws/ini_parser_spec.rb

@@ -29,6 +29,11 @@ module Aws
 
 [sso-session dev]
 sso_region = us-east-1
+
+[services test-services]
+s3 =
+   endpoint_url = https://localhost:8000
+
       FILE
     }
 
@@ -51,6 +56,12 @@ module Aws
 
     it 'can parse sso-session sections' do
       parsed = IniParser.ini_parse(mock_config)
-      expect(parsed['sso-session dev']['sso_region']).to eq('us-east-1')    end
+      expect(parsed['sso-session dev']['sso_region']).to eq('us-east-1')
+    end
+
+    it 'can parse services sections' do
+      parsed = IniParser.ini_parse(mock_config)
+      expect(parsed['services test-services']['s3']['endpoint_url']).to eq('https://localhost:8000')
+    end
   end
 end

gems/aws-sdk-core/spec/aws/plugins/regional_endpoint_spec.rb

@@ -6,14 +6,10 @@ module Aws
   module Plugins
     describe RegionalEndpoint do
       RegionalEndpointClient = ApiHelper.sample_service.const_get(:Client)
-
-      let(:env) { {} }
-
+
       let(:client_class) { RegionalEndpointClient }
 
-      before do
-        stub_const('ENV', env)
-      end
+      let(:region) { 'REGION' }
 
       describe 'region option' do
         it 'adds a :region configuration option' do
@@ -22,35 +18,35 @@ module Plugins
         end
 
         it 'defaults to ENV["AWS_DEFAULT_REGION"]' do
-          env['AWS_DEFAULT_REGION'] = 'env-region'
+          ENV['AWS_DEFAULT_REGION'] = 'env-region'
           expect(client_class.new.config.region).to eq('env-region')
         end
 
         it 'defaults to ENV["AWS_REGION"]' do
-          env['AWS_REGION'] = 'env-fallback1'
+          ENV['AWS_REGION'] = 'env-fallback1'
           expect(client_class.new.config.region).to eq('env-fallback1')
         end
 
         it 'falls back to ENV["AMAZON_REGION"]' do
-          env['AMAZON_REGION'] = 'region-fallback2'
+          ENV['AMAZON_REGION'] = 'region-fallback2'
           expect(client_class.new.config.region).to eq('region-fallback2')
         end
 
         it 'prefers AWS_REGION to AMAZON_REGION or AWS_DEFAULT_REGION' do
-          env['AWS_REGION'] = 'aws-region'
-          env['AMAZON_REGION'] = 'amazon-region'
-          env['AWS_DEFAULT_REGION'] = 'aws-default-region'
+          ENV['AWS_REGION'] = 'aws-region'
+          ENV['AMAZON_REGION'] = 'amazon-region'
+          ENV['AWS_DEFAULT_REGION'] = 'aws-default-region'
           expect(client_class.new.config.region).to eq('aws-region')
         end
 
         it 'prefers AWS_REGION to AMAZON_REGION' do
-          env['AWS_REGION'] = 'aws-region'
-          env['AMAZON_REGION'] = 'amazon-region'
+          ENV['AWS_REGION'] = 'aws-region'
+          ENV['AMAZON_REGION'] = 'amazon-region'
           expect(client_class.new.config.region).to eq('aws-region')
         end
 
         it 'can be set directly, overriding the ENV["AWS_REGION"]' do
-          env['AWS_REGION'] = 'env-region'
+          ENV['AWS_REGION'] = 'env-region'
           expect(client_class.new(region: 'cfg').config.region).to eq('cfg')
         end
 
@@ -90,11 +86,119 @@ module Plugins
       end
 
       describe 'endpoint option' do
-        it 'defaults the endpoint to PREFIX.REGION.amazonaws.com' do
+        it 'preserves legacy pre-endpoints2.0 behavior and sets the endpoint and regional_endpoint' do
           prefix = client_class.api.metadata['endpointPrefix']
-          client = client_class.new(region: 'REGION')
+          client = client_class.new(region: region)
           expect(client.config.endpoint.to_s)
-            .to eq("https://#{prefix}.REGION.amazonaws.com")
+            .to eq("https://#{prefix}.#{region}.amazonaws.com")
+          expect(client.config.regional_endpoint).to be_truthy
+        end
+
+        context 'ENV[AWS_IGNORE_CONFIGURED_ENDPOINT_URLS] set' do
+          before { ENV['AWS_IGNORE_CONFIGURED_ENDPOINT_URLS'] = 'True' }
+
+          it 'ignores ENV[AWS_ENDPOINT_URL]' do
+            ENV['AWS_ENDPOINT_URL'] = 'custom-env-url'
+            expect(client_class.new(region: region).config.endpoint.to_s)
+              .not_to eq('custom-env-url')
+          end
+
+          it 'ignores endpoint_url in shared config' do
+            expect(Aws.shared_config).not_to receive(:configured_endpoint)
+            client_class.new(region: region)
+          end
+
+          it 'uses client configured endpoint' do
+            expect(client_class.new(
+              region: region, endpoint: 'https://custom-client-endpoint'
+            ).config.endpoint.to_s).to eq('https://custom-client-endpoint')
+          end
+        end
+
+        context 'Shared config ignore_configured_endpoint_urls set' do
+          before do
+            allow_any_instance_of(Aws::SharedConfig)
+              .to receive(:ignore_configured_endpoint_urls).and_return('true')
+          end
+
+          it 'ignores ENV[AWS_ENDPOINT_URL]' do
+            ENV['AWS_ENDPOINT_URL'] = 'custom-env-url'
+            expect(client_class.new(region: region).config.endpoint.to_s)
+              .not_to eq('custom-env-url')
+          end
+
+          it 'ignores endpoint_url in shared config' do
+            expect(Aws.shared_config).not_to receive(:configured_endpoint)
+            client_class.new(region: region)
+          end
+
+          it 'uses client configured endpoint' do
+            expect(client_class.new(
+              region: region, endpoint: 'https://custom-client-endpoint'
+            ).config.endpoint.to_s).to eq('https://custom-client-endpoint')
+          end
+        end
+
+        context 'client ignore_configured_endpoint_urls set' do
+          let(:cfg) do
+            {
+              region: region,
+              ignore_configured_endpoint_urls: true
+            }
+          end
+
+          it 'ignores ENV[AWS_ENDPOINT_URL]' do
+            ENV['AWS_ENDPOINT_URL'] = 'custom-env-url'
+            expect(client_class.new(cfg).config.endpoint.to_s)
+              .not_to eq('custom-env-url')
+          end
+
+          it 'ignores endpoint_url in shared config' do
+            expect(Aws.shared_config).not_to receive(:configured_endpoint)
+            client_class.new(cfg)
+          end
+
+          it 'ignores ENV[AWS_ENDPOINT_URL] even when Shared config ignore_configured_endpoint_urls set' do
+            allow_any_instance_of(Aws::SharedConfig)
+              .to receive(:ignore_configured_endpoint_urls).and_return('true')
+            ENV['AWS_ENDPOINT_URL'] = 'custom-env-url'
+            expect(client_class.new(cfg).config.endpoint.to_s)
+              .not_to eq('custom-env-url')
+          end
+        end
+
+        it 'uses configured_endpoint from shared config' do
+          allow(Aws.shared_config).to receive(:configured_endpoint)
+                                        .and_return('https://shared-config')
+          expect(client_class.new(region: region).config.endpoint.to_s)
+            .to eq('https://shared-config')
+        end
+
+        it 'uses ENV[AWS_ENDPOINT_URL] over shared config' do
+          allow(Aws.shared_config).to receive(:configured_endpoint)
+                                        .and_return('https://shared-config')
+          ENV['AWS_ENDPOINT_URL'] = 'https://global-env'
+          expect(client_class.new(region: region).config.endpoint.to_s)
+            .to eq('https://global-env')
+        end
+
+        it 'uses service specific ENV over ENV[AWS_ENDPOINT_URL]' do
+          allow(Aws.shared_config).to receive(:configured_endpoint)
+                                        .and_return('https://shared-config')
+          ENV['AWS_ENDPOINT_URL'] = 'https://global-env'
+          ENV['AWS_ENDPOINT_URL_SVC'] = 'https://service-env'
+          expect(client_class.new(region: region).config.endpoint.to_s)
+            .to eq('https://service-env')
+        end
+
+        it 'uses client configured endpoint over all other configuration' do
+          allow(Aws.shared_config).to receive(:configured_endpoint)
+                                        .and_return('https://shared-config')
+          ENV['AWS_ENDPOINT_URL'] = 'https://global-env'
+          ENV['AWS_ENDPOINT_URL_SVC'] = 'https://service-env'
+          expect(client_class.new(
+            region: region, endpoint: 'https://client').config.endpoint.to_s)
+            .to eq('https://client')
         end
       end