Reduce memory usage from aws-partitions

build_tools/aws-sdk-code-generator/spec/interfaces/plugins/endpoints_spec.rb

@@ -38,6 +38,7 @@
       it 'uses endpoint provider with params to resolve endpoint' do
         expect_any_instance_of(EndpointsPlugin::EndpointProvider)
           .to receive(:resolve_endpoint)
+          .at_least(:once)
           .with(an_instance_of(EndpointsPlugin::EndpointParameters))
           .and_call_original
         client.operation
@@ -124,6 +125,10 @@
         EndpointsPrecedence::Client.new(stub_responses: true, region: 'config')
       end
 
+      before(:each) do
+        client # force client initialization before testing resolution
+      end
+
       # Most to least
       # staticContextParams
       # contextParam

gems/aws-partitions/CHANGELOG.md

@@ -1,7 +1,9 @@
 Unreleased Changes
 ------------------
 
-1.988.0 (2024-10-09)
+* Feature - Add partition metadata module, allowing access without loading entire partitions.json.
+
+* 1.988.0 (2024-10-09)
 ------------------
 
 * Feature - Updated the partitions source data the determines the AWS service regions and endpoints.

gems/aws-partitions/lib/aws-partitions.rb

@@ -5,6 +5,7 @@
 require_relative 'aws-partitions/partition_list'
 require_relative 'aws-partitions/region'
 require_relative 'aws-partitions/service'
+require_relative 'aws-partitions/metadata'
 
 require 'json'
 

gems/aws-partitions/lib/aws-partitions/metadata.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Aws
+  module Partitions
+    # @api private
+    module Metadata
+      class << self
+
+        # aws.partition(region: string) Option<Partition>
+        def partition(region)
+          partition =
+            partitions.find { |p| p['regions']&.fetch(region, nil) } ||
+            partitions.find { |p| region.match(p['regionRegex']) } ||
+            partitions.find { |p| p['id'] == 'aws' }
+
+          return nil unless partition
+
+          partition['outputs']
+        end
+
+        def partitions
+          @partitions ||= default_partition_metadata
+        end
+
+        def default_partition_metadata
+          path = File.expand_path('../../../partitions-metadata.json', __FILE__)
+          JSON.parse(File.read(path), freeze: true)['partitions']
+        end
+      end
+    end
+  end
+end

gems/aws-sdk-core/CHANGELOG.md

@@ -1,6 +1,8 @@
 Unreleased Changes
 ------------------
 
+* Feature - reduce memory usage by not using legacy endpoint data unless required.
+
 3.209.1 (2024-09-25)
 ------------------
 

gems/aws-sdk-core/aws-sdk-core.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |spec|
   spec.files         = Dir['LICENSE.txt', 'CHANGELOG.md', 'VERSION', 'lib/**/*.rb', 'sig/**/*.rbs', 'ca-bundle.crt']
 
   spec.add_dependency('jmespath', '~> 1', '>= 1.6.1') # necessary for secure jmespath JSON parsing
-  spec.add_dependency('aws-partitions', '~> 1', '>= 1.651.0') # necessary for new endpoint resolution
+  spec.add_dependency('aws-partitions', '~> 1', '>= 1.989.0') # necessary for new endpoint resolution
   spec.add_dependency('aws-sigv4', '~> 1.9') # necessary for s3 express auth/native sigv4a support
   spec.add_dependency('aws-eventstream', '~> 1', '>= 1.3.0') # necessary for binary eventstream
 

gems/aws-sdk-core/lib/aws-sdk-core/endpoints/matchers.rb

@@ -94,14 +94,7 @@ def self.valid_host_label?(value, allow_sub_domains = false)
 
       # aws.partition(value: string) Option<Partition>
       def self.aws_partition(value)
-        partition =
-          Aws::Partitions.find { |p| p.region?(value) } ||
-          Aws::Partitions.find { |p| value.match(p.region_regex) } ||
-          Aws::Partitions.find { |p| p.name == 'aws' }
-
-        return nil unless partition
-
-        partition.metadata
+        Aws::Partitions::Metadata.partition(value)
       end
 
       # aws.parseArn(value: string) Option<ARN>

gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb

@@ -20,7 +20,7 @@ class RegionalEndpoint < Seahorse::Client::Plugin
 * `ENV['AWS_DEFAULT_REGION']`
 * `~/.aws/credentials`
 * `~/.aws/config`
-        DOCS
+             DOCS
         resolve_region(cfg)
       end
 
@@ -35,7 +35,7 @@ class RegionalEndpoint < Seahorse::Client::Plugin
 * `Aws.config[:sigv4a_signing_region_set]`
 * `ENV['AWS_SIGV4A_SIGNING_REGION_SET']`
 * `~/.aws/config`
-        DOCS
+             DOCS
         resolve_sigv4a_signing_region_set(cfg)
       end
 
@@ -44,7 +44,7 @@ class RegionalEndpoint < Seahorse::Client::Plugin
         docstring: <<-DOCS) do |cfg|
 When set to `true`, dualstack enabled endpoints (with `.aws` TLD)
 will be used if available.
-        DOCS
+             DOCS
         resolve_use_dualstack_endpoint(cfg)
       end
 
@@ -54,7 +54,7 @@ class RegionalEndpoint < Seahorse::Client::Plugin
 When set to `true`, fips compatible endpoints will be used if available.
 When a `fips` region is used, the region is normalized and this config
 is set to `true`.
-        DOCS
+             DOCS
         resolve_use_fips_endpoint(cfg)
       end
 
@@ -67,22 +67,25 @@ class RegionalEndpoint < Seahorse::Client::Plugin
         docstring: <<-DOCS) do |cfg|
 Setting to true disables use of endpoint URLs provided via environment 
 variables and the shared configuration file.
-        DOCS
+             DOCS
         resolve_ignore_configured_endpoint_urls(cfg)
       end
 
       option(:endpoint, doc_type: String, docstring: <<-DOCS) do |cfg|
 The client endpoint is normally constructed from the `:region`
 option. You should only configure an `:endpoint` when connecting
 to test or custom endpoints. This should be a valid HTTP(S) URI.
-        DOCS
+      DOCS
         resolve_endpoint(cfg)
       end
 
       def after_initialize(client)
         region = client.config.region
         raise Errors::MissingRegionError if region.nil? || region == ''
 
+        # resolve a default endpoint to preserve legacy behavior
+        initialize_default_endpoint(client) if client.config.endpoint.nil?
+
         region_set = client.config.sigv4a_signing_region_set
         return if region_set.nil?
         raise Errors::InvalidRegionSetError unless region_set.is_a?(Array)
@@ -93,6 +96,66 @@ def after_initialize(client)
         client.config.sigv4a_signing_region_set = region_set
       end
 
+      private
+
+      def initialize_default_endpoint(client)
+        client_module = Object.const_get(client.class.name.rpartition('::').first)
+        if client.config.respond_to?(:endpoint_provider) &&
+          client_module.const_defined?(:EndpointParameters) &&
+          (param_class = client_module.const_get(:EndpointParameters)) &&
+          (endpoint_provider = client.config.endpoint_provider)
+
+          params = build_parameters(param_class.new, client.config)
+          begin
+            endpoint = endpoint_provider.resolve_endpoint(params)
+            client.config.endpoint = endpoint.url
+          rescue ArgumentError
+            # fallback to legacy
+            client.config.endpoint = resolve_legacy_endpoint(client.config)
+          end
+        else
+          # fallback to legacy
+          client.config.endpoint = resolve_legacy_endpoint(client.config)
+        end
+      end
+
+      def build_parameters(params, config)
+        # generic parameters set for most (but not all) services
+        params.region = config.region if params.respond_to?(:region=)
+        params.use_dual_stack = config.use_dualstack_endpoint if params.respond_to?(:use_dual_stack=)
+        params.use_fips = config.use_fips_endpoint if params.respond_to?(:use_fips=)
+
+        # service specific
+        if config.respond_to?(:sts_regional_endpoints) && params.respond_to?(:use_global_endpoint=)
+          params.use_global_endpoint = (config.sts_regional_endpoints == 'legacy')
+        end
+
+        if config.respond_to?(:s3_us_east_1_regional_endpoint) && params.respond_to?(:use_global_endpoint=)
+          params.use_global_endpoint = (config.s3_us_east_1_regional_endpoint == 'legacy')
+        end
+
+        params
+      end
+
+      # set a default endpoint in config using legacy (endpoints.json) resolver
+      def resolve_legacy_endpoint(cfg)
+        endpoint_prefix = cfg.api.metadata['endpointPrefix']
+        if cfg.respond_to?(:sts_regional_endpoints)
+          sts_regional = cfg.sts_regional_endpoints
+        end
+
+        endpoint = Aws::Partitions::EndpointProvider.resolve(
+          cfg.region,
+          endpoint_prefix,
+          sts_regional,
+          {
+            dualstack: cfg.use_dualstack_endpoint,
+            fips: cfg.use_fips_endpoint
+          }
+        )
+        URI(endpoint)
+      end
+
       class << self
         private
 
@@ -150,7 +213,8 @@ def resolve_endpoint(cfg)
           # that a custom endpoint has NOT been configured by the user
           cfg.override_config(:regional_endpoint, true)
 
-          resolve_legacy_endpoint(cfg)
+          # a default endpoint is resolved in after_initialize
+          nil
         end
 
         # get a custom configured endpoint from ENV or configuration
@@ -205,24 +269,6 @@ def handle_legacy_pseudo_regions(cfg)
             cfg.override_config(:region, new_region)
           end
         end
-
-        # set a default endpoint in config using legacy (endpoints.json) resolver
-        def resolve_legacy_endpoint(cfg)
-          endpoint_prefix = cfg.api.metadata['endpointPrefix']
-          if cfg.respond_to?(:sts_regional_endpoints)
-            sts_regional = cfg.sts_regional_endpoints
-          end
-
-          Aws::Partitions::EndpointProvider.resolve(
-            cfg.region,
-            endpoint_prefix,
-            sts_regional,
-            {
-              dualstack: cfg.use_dualstack_endpoint,
-              fips: cfg.use_fips_endpoint
-            }
-          )
-        end
       end
     end
   end

gems/aws-sdk-core/spec/aws/endpoints_spec.rb

@@ -66,7 +66,7 @@ module Aws
 
     before do
       expect_any_instance_of(endpoints_service.const_get(:EndpointProvider))
-        .to receive(:resolve_endpoint).and_return(endpoint)
+        .to receive(:resolve_endpoint).at_least(:once).and_return(endpoint)
     end
 
     describe '#resolve_auth_scheme' do