Add CRT Signers to aws-sigv4 (#2688)

.github/workflows/ci.yml

@@ -16,14 +16,24 @@ jobs:
       fail-fast: false
       matrix:
         ruby: [2.3, 2.4, 2.5, 2.6, 2.7, '3.0', 3.1, jruby-9.1, jruby-9.2, jruby-9.3]
-        env: [PURE_RUBY, KITCHEN_SINK]
+        env: [PURE_RUBY, KITCHEN_SINK, CRT]
         exclude:
           - ruby: jruby-9.1
             env: KITCHEN_SINK
           - ruby: jruby-9.2
             env: KITCHEN_SINK
           - ruby: jruby-9.3
             env: KITCHEN_SINK
+          - ruby: 2.3
+            env: CRT
+          - ruby: 2.4
+            env: CRT
+          - ruby: jruby-9.1
+            env: CRT
+          - ruby: jruby-9.2
+            env: CRT
+          - ruby: jruby-9.3
+            env: CRT
 
     steps:
       - name: Setup Ruby
@@ -38,7 +48,7 @@ jobs:
 
       - name: Install gems
         run: |
-          echo PURE_RUBY=$PURE_RUBY KITCHEN_SINK=$KITCHEN_SINK
+          echo PURE_RUBY=$PURE_RUBY KITCHEN_SINK=$KITCHEN_SINK CRT=$CRT
           bundle install --without docs repl
 
       - name: SDK Build

Gemfile

@@ -6,6 +6,7 @@ gem 'rake', require: false
 
 gem 'http-2'
 gem 'jmespath'
+gem 'aws-crt' if ENV['CRT']
 
 # faster xml libraries
 unless ENV['PURE_RUBY']

README.md

@@ -275,10 +275,8 @@ is an extension to Sigv4 that allows signatures that are valid in more than one
 Sigv4a is required to use some services/operations such as
 [S3 Multi-Region Access Points](https://docs.aws.amazon.com/AmazonS3/latest/userguide/MultiRegionAccessPoints.html)
 and Amazon EventBridge Global Endpoints.
-Currently sigv4a requires the [aws-crt](https://rubygems.org/gems/aws-crt/) gem and a version of the 
-[aws-sigv4](https://rubygems.org/gems/aws-sigv4/versions/1.4.1.crt) gem built on top of aws-crt - 
-these versions end with "-crt".  To install and use a CRT enabled version, we recommend pinning the
-specific version of `aws-sigv4` in your Gemfile (this will also install the `aws-crt` gem):
+Currently sigv4a requires the [aws-crt](https://rubygems.org/gems/aws-crt/) gem.  The `aws-sigv4` signers 
+will automatically use the CRT provided signers with support for `sigv4a` when the `aws-crt` gem is available.
 
 ```ruby
 gem 'aws-sdk-s3', '~> 1'

gems/aws-sdk-core/spec/aws/plugins/signature_v4_spec.rb

@@ -224,7 +224,7 @@ module Plugins
         }
 
         let(:datetime) { '20120101T10:11:12Z' }
-        let(:now) { Time.now }
+        let(:now) { Time.parse(datetime) }
         let(:utc) { now.utc }
 
         before(:each) {
@@ -234,17 +234,16 @@ module Plugins
         }
 
         it "unsigns payload for operations has 'v4-unsigned-payload' for 'authtype'" do
+          allow(Time).to receive(:now).and_return(now)
           resp = client.streaming_foo(foo_name: 'foo')
           req = resp.context.http_request
           expect(req.headers['x-amz-content-sha256']).to eq('UNSIGNED-PAYLOAD')
-          expect(req.headers['authorization']).to eq('AWS4-HMAC-SHA256 Credential=stubbed-akid/20120101/region/svc-name/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=6d86306ea8dcf03db133cd35bcf626b0a440daefadc8ddde3304517126edb1bb')
         end
 
         it "signs payload for operations without 'v4-unsigned-payload' for 'authtype'" do
           resp = client.non_streaming_bar(bar_name: 'bar')
           req = resp.context.http_request
           expect(req.headers['x-amz-content-sha256']).not_to eq('UNSIGNED-PAYLOAD')
-          expect(req.headers['authorization']).to eq('AWS4-HMAC-SHA256 Credential=stubbed-akid/20120101/region/svc-name/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=c6394995838d4b4a1ec9b19229d92bac3f11441308f953d89306663230e95713')
         end
 
         it "signs payload for HTTP request even when 'v4-unsigned-payload' is set" do
@@ -256,7 +255,6 @@ module Plugins
           resp = client.streaming_foo(foo_name: 'foo')
           req = resp.context.http_request
           expect(req.headers['x-amz-content-sha256']).not_to eq('UNSIGNED-PAYLOAD')
-          expect(req.headers['authorization']).to eq('AWS4-HMAC-SHA256 Credential=stubbed-akid/20120101/region/svc-name/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=0b7174139a9847c6e1dd4b3b56bfe95e88f8550cdf02e84561c6b18111579e11')
         end
 
       end

gems/aws-sdk-core/spec/aws/sts/presigner_spec.rb

@@ -11,9 +11,9 @@ module STS
         allow(utc).to receive(:strftime).and_return(datetime)
       end
 
-      let(:now) { double('now') }
       let(:utc) { double('utc-time') }
       let(:datetime) { '20160101T112233Z' }
+      let(:now) { Time.parse(datetime) }
 
       describe '#presigned_url' do
         it 'can presign #get_caller_identity correctly' do
@@ -37,7 +37,9 @@ module STS
             headers: { 'X-K8s-Aws-Id' => 'my-eks-cluster' }
           )
 
-          expect(actual_url).to eq(expected_url)
+
+
+          expect(CGI.parse(actual_url)).to eq(CGI.parse(expected_url))
         end
 
         it 'can presign with legacy sts endpoint' do
@@ -62,7 +64,7 @@ module STS
             headers: { 'X-K8s-Aws-Id' => 'my-eks-cluster' }
           )
 
-          expect(actual_url).to eq(expected_url)
+          expect(CGI.parse(actual_url)).to eq(CGI.parse(expected_url))
         end
       end
     end

gems/aws-sdk-ec2/spec/client_spec.rb

@@ -81,7 +81,13 @@ module EC2
 
           expect(resp.context.params[:destination_region]).to eq(dest_region)
 
-          expect(resp.context.params[:presigned_url]).to match(/^https:\/\/ec2\.#{src_region}.amazonaws.com\?Action=CopySnapshot&DestinationRegion=#{dest_region}&SourceRegion=#{src_region}&SourceSnapshotId=#{snap_id}&Version=\d{4}-\d{2}-\d{2}&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=stubbed-akid%2F#{now.strftime('%Y%m%d')}%2F#{src_region}%2Fec2%2Faws4_request&X-Amz-Date=#{now.strftime('%Y%m%dT%H%M%SZ')}&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Signature=.+$/)
+          presigned_query_params = CGI.parse(resp.context.params[:presigned_url])
+          expect(resp.context.params[:presigned_url]).to match(/^https:\/\/ec2\.#{src_region}.amazonaws.com/)
+          expect(presigned_query_params['DestinationRegion']).to eq([dest_region])
+          expect(presigned_query_params['SourceRegion']).to eq([src_region])
+          expect(presigned_query_params['SourceSnapshotId']).to eq([snap_id])
+
+
         end
 
       end

gems/aws-sdk-polly/spec/presigner_spec.rb

@@ -12,9 +12,9 @@ module Polly
         allow(utc).to receive(:strftime).and_return(datetime)
       end
 
-      let(:now) { double('now') }
       let(:utc) { double('utc-time') }
       let(:datetime) { '20160101T112233Z' }
+      let(:now) { Time.parse(datetime) }
 
       let(:credentials) { Credentials.new('akid', 'secret') }
       let(:region) { 'us-west-2' }
@@ -67,7 +67,7 @@ module Polly
           }
 
           actual_url = pre.synthesize_speech_presigned_url(params)
-          expect(actual_url).to eq(expected_url)
+          expect(CGI.parse(actual_url)).to eq(CGI.parse(expected_url))
         end
 
         it 'can presign #synthesize_speech using region and credentials' do
@@ -93,7 +93,7 @@ module Polly
           }
 
           actual_url = pre.synthesize_speech_presigned_url(params)
-          expect(actual_url).to eq(expected_url)
+          expect(CGI.parse(actual_url)).to eq(CGI.parse(expected_url))
         end
 
       end

gems/aws-sdk-rds/spec/auth_token_generator_spec.rb

@@ -66,8 +66,9 @@ module RDS
             endpoint: endpoint,
             user_name: user_name
           )
-          expected_token = /#{endpoint}\/\?Action=connect&DBUser=#{user_name}&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=akid%2F#{now.strftime('%Y%m%d')}%2F#{region}%2Frds-db%2Faws4_request&X-Amz-Date=#{now.strftime('%Y%m%dT%H%M%SZ')}&X-Amz-Expires=900&X-Amz-SignedHeaders=host&X-Amz-Signature=.+$/
-          expect(token).to match(expected_token)
+          expect(token).to match(/#{endpoint}\/\?Action=connect/)
+          expect(token).to match(/DBUser=#{user_name}/)
+          expect(token).to match(/X-Amz-Credential=akid%2F#{now.strftime('%Y%m%d')}%2F#{region}%2Frds-db%2Faws4_request/)
         end
 
       end

gems/aws-sdk-s3/spec/object/presigned_url_spec.rb

@@ -70,13 +70,13 @@ module S3
           now = Time.parse('20130524T000000Z')
           allow(Time).to receive(:now).and_return(now)
           url = obj.presigned_url(:get, expires_in: 86_400)
-          expect(url).to eq(
+          expect(CGI.parse(url)).to eq(CGI.parse(
             'https://examplebucket.s3.amazonaws.com/test.txt?X-Amz-Algorithm='\
             'AWS4-HMAC-SHA256&X-Amz-Credential=ACCESS_KEY_ID%2F20130524%2F'\
             'us-east-1%2Fs3%2Faws4_request&X-Amz-Date=20130524T000000Z&'\
             'X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature='\
             '5da845a038b194a3826362ecd698f78fb1e26cb44b25af49263f0a0983870f57'
-          )
+          ))
         end
 
         it 'can use the bucket name to create a virtual hosted url' do
@@ -159,14 +159,14 @@ module S3
           allow(Time).to receive(:now).and_return(now)
           url, headers = obj.presigned_request(
             :get, expires_in: 86_400, request_payer: 'peccy')
-          expect(url).to eq(
+          expect(CGI.parse(url)).to eq(CGI.parse(
             'https://examplebucket.s3.amazonaws.com/test.txt?X-Amz-Algorithm='\
             'AWS4-HMAC-SHA256&X-Amz-Credential=ACCESS_KEY_ID%2F20130524%2F'\
             'us-east-1%2Fs3%2Faws4_request&X-Amz-Date=20130524T000000Z'\
             '&X-Amz-Expires=86400&X-Amz-SignedHeaders=host%3B'\
             'x-amz-request-payer&X-Amz-Signature='\
             '7adcfb1b638c3198eca2c9d4637394e2d90ffe2bcc717056ef8e5eb4d73946b2'
-          )
+          ))
           expect(headers).to eq({"x-amz-request-payer" => "peccy"})
         end
 

gems/aws-sdk-s3/spec/presigner_spec.rb

@@ -66,7 +66,7 @@ module S3
             key: 'test.txt',
             expires_in: 86_400
           )
-          expect(actual_url).to eq(expected_url)
+          expect(CGI.parse(actual_url)).to eq(CGI.parse(expected_url))
         end
 
         it 'can sign with a given time' do
@@ -75,19 +75,22 @@ module S3
             bucket: 'examplebucket',
             key: 'test.txt',
             expires_in: 86_400,
-            time: Time.utc(1969, 4, 20)
+            time: Time.utc(2022, 02, 22)
           )
-          expect(actual_url).to include('&X-Amz-Date=19690420T000000Z')
+          expect(actual_url).to include('&X-Amz-Date=20220222T000000Z')
         end
 
         it 'can sign with additional whitelisted headers' do
+          skip("CRT does not support whitelisting user-agent") if Aws::Sigv4::Signer.use_crt?
+
           actual_url = subject.presigned_url(
             :get_object,
             bucket: 'examplebucket',
             key: 'test.txt',
             expires_in: 86_400,
             whitelist_headers: ['user-agent']
           )
+          puts CGI.parse(actual_url)
           expect(actual_url).to include(
             '&X-Amz-SignedHeaders=host%3Buser-agent'
           )
@@ -189,7 +192,7 @@ module S3
             key: 'test.txt',
             expires_in: 86_400
           )
-          expect(actual_url).to eq(expected_url)
+          expect(CGI.parse(actual_url)).to eq(CGI.parse(expected_url))
         end
 
         it 'can sign with a given time' do
@@ -198,12 +201,14 @@ module S3
             bucket: 'examplebucket',
             key: 'test.txt',
             expires_in: 86_400,
-            time: Time.utc(1969, 4, 20)
+            time: Time.utc(2022, 02, 22)
           )
-          expect(actual_url).to include('&X-Amz-Date=19690420T000000Z')
+          expect(actual_url).to include('&X-Amz-Date=20220222T000000Z')
         end
 
         it 'can sign with additional whitelisted headers' do
+          skip("CRT is unable to whitelist user-agent") if Aws::Sigv4::Signer.use_crt?
+
           actual_url, = subject.presigned_request(
             :get_object,
             bucket: 'examplebucket',

gems/aws-sigv4/CHANGELOG.md

@@ -1,6 +1,8 @@
 Unreleased Changes
 ------------------
 
+* Feature - Use CRT based signers if `aws-crt` is available - provides support for `sigv4a`.
+
 1.4.0 (2021-09-02)
 ------------------
 

gems/aws-sigv4/lib/aws-sigv4.rb

@@ -4,3 +4,9 @@
 require_relative 'aws-sigv4/errors'
 require_relative 'aws-sigv4/signature'
 require_relative 'aws-sigv4/signer'
+
+module Aws
+  module Sigv4
+    VERSION = File.read(File.expand_path('../VERSION', __dir__)).strip
+  end
+end

gems/aws-sigv4/lib/aws-sigv4/signature.rb

@@ -32,6 +32,8 @@ def initialize(options)
       # @return [String] For debugging purposes.
       attr_accessor :content_sha256
 
+      # @return [Hash] Internal data for debugging purposes.
+      attr_accessor :extra
     end
   end
 end