From dd8f32968a26946db3bfc2beb0e9ef24a926e3a9 Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Mon, 26 Aug 2024 10:30:14 -0700 Subject: [PATCH] Reduce initial memory usage by auto-loading bundled gems (2) (#3088) --- gems/aws-sdk-core/CHANGELOG.md | 2 ++ gems/aws-sdk-core/lib/aws-sdk-core.rb | 21 ++++++++++++--------- gems/aws-sdk-core/lib/aws-sdk-sso.rb | 6 +++++- gems/aws-sdk-core/lib/aws-sdk-ssooidc.rb | 6 +++++- gems/aws-sdk-core/lib/aws-sdk-sts.rb | 6 +++++- gems/aws-sdk-resources/CHANGELOG.md | 2 ++ gems/aws-sdk-resources/bin/aws-v3.rb | 6 ++++-- tasks/build.rake | 3 +++ tasks/update-aws-sdk-dependencies.rake | 4 +++- 9 files changed, 41 insertions(+), 15 deletions(-) diff --git a/gems/aws-sdk-core/CHANGELOG.md b/gems/aws-sdk-core/CHANGELOG.md index 54b7503d65b..17cc4f4595d 100644 --- a/gems/aws-sdk-core/CHANGELOG.md +++ b/gems/aws-sdk-core/CHANGELOG.md @@ -1,6 +1,8 @@ Unreleased Changes ------------------ +* Issue - Reduce initial memory usage by auto-loading bundled gems (STS, SSO, SSOOIDC). + 3.201.5 (2024-08-15) ------------------ diff --git a/gems/aws-sdk-core/lib/aws-sdk-core.rb b/gems/aws-sdk-core/lib/aws-sdk-core.rb index f9685d2373b..4beb4c769b5 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core.rb @@ -109,15 +109,7 @@ # defaults require_relative 'aws-defaults' -# plugins -# loaded through building STS or SSO .. - -# aws-sdk-sts is included to support Aws::AssumeRoleCredentials -require_relative 'aws-sdk-sts' - -# aws-sdk-sso is included to support Aws::SSOCredentials -require_relative 'aws-sdk-sso' -require_relative 'aws-sdk-ssooidc' +# plugins - loaded through service clients as needed. module Aws @@ -205,3 +197,14 @@ def eager_autoload!(*args) end end + +# Autoload bundled service gems used in credential providers +# need to ensure that the files are the local files from aws-sdk-core +# and not the installed, legacy/dummy service gems. + +# aws-sdk-sts is included to support Aws::AssumeRoleCredentials +Aws.autoload :STS, File.join(__dir__, 'aws-sdk-sts.rb') + +# aws-sdk-sso is included to support Aws::SSOCredentials +Aws.autoload :SSO, File.join(__dir__, 'aws-sdk-sso.rb') +Aws.autoload :SSOOIDC, File.join(__dir__, 'aws-sdk-ssooidc.rb') diff --git a/gems/aws-sdk-core/lib/aws-sdk-sso.rb b/gems/aws-sdk-core/lib/aws-sdk-sso.rb index 47e3ade88c7..ff232ba6a3b 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-sso.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-sso.rb @@ -7,12 +7,16 @@ # # WARNING ABOUT GENERATED CODE - unless Module.const_defined?(:Aws) require 'aws-sdk-core' require 'aws-sigv4' end +if Aws.autoload?(:SSO) && + !(defined?(JRUBY_VERSION) && JRUBY_VERSION < '9.3') + Aws.autoload(:SSO, __FILE__) +end + require_relative 'aws-sdk-sso/types' require_relative 'aws-sdk-sso/client_api' require_relative 'aws-sdk-sso/plugins/endpoints.rb' diff --git a/gems/aws-sdk-core/lib/aws-sdk-ssooidc.rb b/gems/aws-sdk-core/lib/aws-sdk-ssooidc.rb index 4df75d19e33..fcc3e0f3306 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-ssooidc.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-ssooidc.rb @@ -7,12 +7,16 @@ # # WARNING ABOUT GENERATED CODE - unless Module.const_defined?(:Aws) require 'aws-sdk-core' require 'aws-sigv4' end +if Aws.autoload?(:SSOOIDC) && + !(defined?(JRUBY_VERSION) && JRUBY_VERSION < '9.3') + Aws.autoload(:SSOOIDC, __FILE__) +end + require_relative 'aws-sdk-ssooidc/types' require_relative 'aws-sdk-ssooidc/client_api' require_relative 'aws-sdk-ssooidc/plugins/endpoints.rb' diff --git a/gems/aws-sdk-core/lib/aws-sdk-sts.rb b/gems/aws-sdk-core/lib/aws-sdk-sts.rb index 2330c3c2934..6811cd5387a 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-sts.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-sts.rb @@ -7,12 +7,16 @@ # # WARNING ABOUT GENERATED CODE - unless Module.const_defined?(:Aws) require 'aws-sdk-core' require 'aws-sigv4' end +if Aws.autoload?(:STS) && + !(defined?(JRUBY_VERSION) && JRUBY_VERSION < '9.3') + Aws.autoload(:STS, __FILE__) +end + require_relative 'aws-sdk-sts/types' require_relative 'aws-sdk-sts/client_api' require_relative 'aws-sdk-sts/plugins/endpoints.rb' diff --git a/gems/aws-sdk-resources/CHANGELOG.md b/gems/aws-sdk-resources/CHANGELOG.md index f0e4a20fcba..8bffd642bb3 100644 --- a/gems/aws-sdk-resources/CHANGELOG.md +++ b/gems/aws-sdk-resources/CHANGELOG.md @@ -1,6 +1,8 @@ Unreleased Changes ------------------ +* Issue - Remove autoloads for service gems bundled in `aws-sdk-core` (STS, SSO, SSOOIDC). + 3.201.0 (2024-08-12) ------------------ diff --git a/gems/aws-sdk-resources/bin/aws-v3.rb b/gems/aws-sdk-resources/bin/aws-v3.rb index 9b77e69ea72..44886eef2c4 100755 --- a/gems/aws-sdk-resources/bin/aws-v3.rb +++ b/gems/aws-sdk-resources/bin/aws-v3.rb @@ -106,11 +106,13 @@ def env_bool key, default # The Aws namespace used to check autoload requires aws-sdk-resources. require 'aws-sdk-resources' -# Finally load all of the gems. Core is loaded a second time because of STS. +# Finally load all of the gems. +# Don't update the load path for gems bundled in core. +core_gems = [:STS, :SSO, :SSOOIDC] if File.directory?(File.expand_path('../../../../build_tools', __FILE__)) gems = [] Aws.constants.each do |const_name| - if Aws.autoload?(const_name) + if Aws.autoload?(const_name) && !core_gems.include?(const_name) gems << "aws-sdk-#{const_name.downcase}" end end diff --git a/tasks/build.rake b/tasks/build.rake index 8767374fc34..138d4f2c8e1 100644 --- a/tasks/build.rake +++ b/tasks/build.rake @@ -28,6 +28,7 @@ end # It is need to provide session credentials and assume role support. # Only building source, but not gemspecs, version file, etc. task 'build:aws-sdk-sts' do + Aws::STS if Aws.autoload?(:STS) # force autoload from core sts = BuildTools::Services.service('sts') generator = AwsSdkCodeGenerator::CodeBuilder.new( aws_sdk_core_lib_path: $CORE_LIB, @@ -48,6 +49,7 @@ end # It is need to provide SSO Credentials. # Only building source, but not gemspecs, version file, etc. task 'build:aws-sdk-sso' do + Aws::SSO if Aws.autoload?(:SSO) # force autoload from core sso = BuildTools::Services.service('sso') generator = AwsSdkCodeGenerator::CodeBuilder.new( aws_sdk_core_lib_path: $CORE_LIB, @@ -66,6 +68,7 @@ end # Aws::SSOOIDC is generated directly into the `aws-sdk-core` gem. # Only building source, but not gemspecs, version file, etc. task 'build:aws-sdk-ssooidc' do + Aws::SSOOIDC if Aws.autoload?(:SSOOIDC) # force autoload from core ssooidc = BuildTools::Services.service('ssooidc') generator = AwsSdkCodeGenerator::CodeBuilder.new( aws_sdk_core_lib_path: $CORE_LIB, diff --git a/tasks/update-aws-sdk-dependencies.rake b/tasks/update-aws-sdk-dependencies.rake index c3c11e494f5..bbd8774304d 100644 --- a/tasks/update-aws-sdk-dependencies.rake +++ b/tasks/update-aws-sdk-dependencies.rake @@ -19,11 +19,13 @@ task 'update-aws-sdk-dependencies' do ) # update the module autoloads + # Remove service gems bundled in core (eg, sts) that have gem_name of aws-sdk-core. + autoload_services = BuildTools::Services.select { |svc| svc.gem_name != 'aws-sdk-core' } BuildTools.replace_lines( filename: "#{$GEMS_DIR}/aws-sdk-resources/lib/aws-sdk-resources.rb", start: /# service gems/, stop: /# end service gems/, - new_lines: BuildTools::Services.map { |service| + new_lines: autoload_services.map { |service| " autoload :#{service.name}, '#{service.gem_name}'\n" } )