Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

SigV4 as Default for All Amazon S3 Calls #979

Merged
merged 4 commits into from
Nov 12, 2015
Merged

SigV4 as Default for All Amazon S3 Calls #979

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3e00562
SigV4 as Default for All Amazon S3 Calls
awood45 Oct 29, 2015
7b3e1c2
Remove Debug Statement from Cucumber Test
awood45 Nov 12, 2015
c72f981
S3 Signer Error Message Improvement + CR Changes
awood45 Nov 12, 2015
c10c1aa
Change US Standard Endpoint for Aws::S3
awood45 Nov 12, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
146 changes: 32 additions & 114 deletions aws-sdk-core/lib/aws-sdk-core/plugins/s3_request_signer.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,29 +6,18 @@ module Plugins
# @api private
class S3RequestSigner < Seahorse::Client::Plugin

class SigningHandler < RequestSigner::Handler
option(:signature_version, 'v4')

# List of regions that support older S3 signature versions.
# All new regions only support signature version 4.
V2_REGIONS = Set.new(%w(
us-east-1
us-west-1
us-west-2
ap-northeast-1
ap-southeast-1
ap-southeast-2
sa-east-1
eu-west-1
us-gov-west-1
))
class SigningHandler < RequestSigner::Handler

def call(context)
require_credentials(context)
version = signature_version(context)
case version
when /v4/ then apply_v4_signature(context)
when /s3/ then apply_v2_signature(context)
else raise "unsupported signature version #{version.inspect}"
case context.config.signature_version
when 'v4' then apply_v4_signature(context)
when 's3' then apply_s3_legacy_signature(context)
else
raise "unsupported signature version #{version.inspect}, valid"\
" options: 'v4' (default), 's3'"
end
@handler.call(context)
end
Expand All @@ -42,59 +31,10 @@ def apply_v4_signature(context)
).sign(context.http_request)
end

def apply_v2_signature(context)
def apply_s3_legacy_signature(context)
Signers::S3.sign(context)
end

def signature_version(context)
context[:cached_signature_version] ||
context.config.signature_version ||
version_by_region(context)
end

def version_by_region(context)
if classic_endpoint?(context)
classic_sigv(context)
else
regional_sigv(context)
end
end

def classic_endpoint?(context)
context.config.region == 'us-east-1'
end

# When accessing the classic endpoint, s3.amazonaws.com, we don't know
# the region name. This makes creating a version 4 signature difficult.
# Choose v4 only if using KMS encryptions, which requires v4.
def classic_sigv(context)
if kms_encrypted?(context)
:v4
else
:s3
end
end

def regional_sigv(context)
# Drop back to older S3 signature version when uploading objects for
# better performance. This optimization may be removed at some point
# in favor of always using signature version 4.
if V2_REGIONS.include?(context.config.region)
uploading_file?(context) && !kms_encrypted?(context) ? :s3 : :v4
else
:v4
end
end

def kms_encrypted?(context)
context.params[:server_side_encryption] == 'aws:kms'
end

def uploading_file?(context)
[:put_object, :upload_part].include?(context.operation_name) &&
context.http_request.body.size > 0
end

end

# Abstract base class for the other two handlers
Expand All @@ -105,7 +45,7 @@ class Handler < Seahorse::Client::Handler
def new_hostname(context, region)
bucket = context.params[:bucket]
if region == 'us-east-1'
"#{bucket}.s3-external-1.amazonaws.com"
"#{bucket}.s3.amazonaws.com"
else
bucket + '.' + URI.parse(EndpointProvider.resolve(region, 's3')).host
end
Expand Down Expand Up @@ -137,13 +77,11 @@ def use_regional_endpoint_when_known(context, bucket)

end

# This handler detects when a request fails because signature version 4
# is required but not used. It follows up by making a request to
# determine the correct region, then finally a version 4 signed
# request against the regional endpoint.
class BucketSigningErrorHandler < Handler

SIGV4_MSG = /(Please use AWS4-HMAC-SHA256|AWS Signature Version 4)/
# This handler detects when a request fails because of a mismatched bucket
# region. It follows up by making a request to determine the correct
# region, then finally a version 4 signed request against the correct
# regional endpoint.
class BucketRegionErrorHandler < Handler

def call(context)
response = @handler.call(context)
Expand All @@ -153,52 +91,34 @@ def call(context)
private

def handle_region_errors(response)
if sigv4_required_error?(response)
detect_region_and_retry(response)
elsif wrong_sigv4_region?(response)
extract_body_region_and_retry(response.context)
if wrong_sigv4_region?(response)
get_region_and_retry(response.context)
else
response
end
end

def sigv4_required_error?(resp)
resp.context.http_response.status_code == 400 &&
resp.context.http_response.body_contents.match(SIGV4_MSG) &&
resp.context.http_response.body.respond_to?(:truncate)
end

def wrong_sigv4_region?(resp)
resp.context.http_response.status_code == 400 &&
resp.context.http_response.body_contents.match(/<Region>.+?<\/Region>/)
end

def extract_body_region_and_retry(context)
def get_region_and_retry(context)
actual_region = region_from_body(context)
updgrade_to_v4(context, actual_region)
if actual_region.nil? || actual_region == ""
raise "Couldn't get region from body: #{context.body}"
end
update_bucket_cache(context, actual_region)
log_warning(context, actual_region)
update_region_header(context, actual_region)
@handler.call(context)
end

def region_from_body(context)
context.http_response.body_contents.match(/<Region>(.+?)<\/Region>/)[1]
def update_bucket_cache(context, actual_region)
S3::BUCKET_REGIONS[context.params[:bucket]] = actual_region
end

def detect_region_and_retry(resp)
context = resp.context
updgrade_to_v4(context, 'us-east-1')
resp = @handler.call(context)
if resp.successful?
resp
else
actual_region = region_from_location_header(context)
updgrade_to_v4(context, actual_region)
log_warning(context, actual_region)
@handler.call(context)
end
def wrong_sigv4_region?(resp)
resp.context.http_response.status_code == 400 &&
resp.context.http_response.body_contents.match(/<Region>.+?<\/Region>/)
end

def updgrade_to_v4(context, region)
def update_region_header(context, region)
context.http_response.body.truncate(0)
context.http_request.headers.delete('authorization')
context.http_request.headers.delete('x-amz-security-token')
Expand All @@ -207,13 +127,11 @@ def updgrade_to_v4(context, region)
signer.sign(context.http_request)
end

def region_from_location_header(context)
location = context.http_response.headers['location']
location.match(/s3[.-](.+?)\.amazonaws\.com/)[1]
def region_from_body(context)
context.http_response.body_contents.match(/<Region>(.+?)<\/Region>/)[1]
end

def log_warning(context, actual_region)
S3::BUCKET_REGIONS[context.params[:bucket]] = actual_region
msg = "S3 client configured for #{context.config.region.inspect} " +
"but the bucket #{context.params[:bucket].inspect} is in " +
"#{actual_region.inspect}; Please configure the proper region " +
Expand All @@ -234,7 +152,7 @@ def log_warning(context, actual_region)
handler(SigningHandler, step: :sign)

# AFTER signing
handle(BucketSigningErrorHandler, step: :sign, priority: 40)
handle(BucketRegionErrorHandler, step: :sign, priority: 40)

end
end
Expand Down
70 changes: 2 additions & 68 deletions aws-sdk-core/spec/aws/s3/client/region_detection_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,72 +17,6 @@ module S3
S3::BUCKET_REGIONS.clear
end

describe 'accessing eu-central-1 via classic endpoint and sigv2' do

before(:each) do

stub_request(:put, 'https://bucket.s3.amazonaws.com/key').
to_return(status: [400, 'Bad Request'], body: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>InvalidRequest</Code><Message>The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.</Message><RequestId>164473A33C2BACDF</RequestId><HostId>I8hhmfGvN74WPyu6kduBoVhnHPgKATFF4nWlro8kRdpuFxXVgcRgLZ2oTxJFhZMgAn75GqyhUHY=</HostId></Error>")

stub_request(:put, 'https://bucket.s3-external-1.amazonaws.com/key').
to_return(status: [307, 'Temporary Redirect'], headers: {
'Location' => 'https://bucket.s3.eu-central-1.amazonaws.com/key'
})

stub_request(:put, 'https://bucket.s3.eu-central-1.amazonaws.com/key').
to_return(status: [200, 'Ok'])

end

it 'detects the sigv4 error, determines actual region and redirects' do
resp = client.put_object(bucket:'bucket', key:'key', body:'body')
host = resp.context.http_request.endpoint.host
expect(host).to eq('bucket.s3.eu-central-1.amazonaws.com')
end

it 'uses cached regions to determine the actual region' do
client.put_object(bucket:'bucket', key:'key', body:'body')
client.put_object(bucket:'bucket', key:'key', body:'body')
client.put_object(bucket:'bucket', key:'key', body:'body')
end

it 'sends a warning to stderr' do
expect($stderr).to receive(:write).with(<<-WARNING.strip + "\n")
S3 client configured for "us-east-1" but the bucket "bucket" is in "eu-central-1"; Please configure the proper region to avoid multiple unecessary redirects and signing attempts
WARNING
client.put_object(bucket:'bucket', key:'key', body:'body')
end

it 'sends the warning to the logger if configured' do
logger = double('logger').as_null_object
expect(logger).to receive(:warn).with(<<-WARNING.strip + "\n")
S3 client configured for "us-east-1" but the bucket "bucket" is in "eu-central-1"; Please configure the proper region to avoid multiple unecessary redirects and signing attempts
WARNING
client = S3::Client.new(client_opts.merge(logger: logger))
client.put_object(bucket:'bucket', key:'key', body:'body')
end

it 'triggers a callback where you can listen for new bucket regions' do
yielded = nil
S3::BUCKET_REGIONS.bucket_added {|*args| yielded = args }
client.put_object(bucket:'bucket', key:'key', body:'body')
expect(yielded).to eq(['bucket', 'eu-central-1'])
end

it 'triggers a callback where you can listen for new bucket regions' do
yielded = nil
S3::BUCKET_REGIONS.bucket_added {|*args| yielded = args }
client.put_object(bucket:'bucket', key:'key', body:'body')
expect(yielded).to eq(['bucket', 'eu-central-1'])
expect(S3::BUCKET_REGIONS.to_h).to eq('bucket' => 'eu-central-1')

expect {
S3::BUCKET_REGIONS.bucket_added
}.to raise_error(ArgumentError, 'missing required block')
end

end

describe 'accessing a bucket in eu-central-1 with wrong region' do

before(:each) do
Expand All @@ -102,7 +36,7 @@ module S3

end

describe 'accessing eu-central-1 bucket using classic endpoitn and wrong sigv4 region' do
describe 'accessing eu-central-1 bucket using classic endpoint and wrong sigv4 region' do

before(:each) do

Expand All @@ -116,7 +50,7 @@ module S3

it 'detects the moved permanently and redirects' do
client = S3::Client.new(client_opts.merge(
region: 'us-west-2', signature_version:'v4')
region: 'us-west-2')
)
resp = client.put_object(bucket:'bucket', key:'key', body:'body')
host = resp.context.http_request.endpoint.host
Expand Down
40 changes: 1 addition & 39 deletions aws-sdk-core/spec/aws/s3/client_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,37 +102,6 @@ module S3
"sa-east-1",
"eu-west-1",
"us-gov-west-1",
].each do |region|

it "defaults signature version 4 for #{region}" do
client = Client.new(stub_responses: true, region: region)
resp = client.head_object(bucket:'name', key:'key')
expect(resp.context.http_request.headers['authorization']).to match(
'AWS4-HMAC-SHA256')
end

it "falls back on classic S3 signing for #put_object in #{region}" do
client = Client.new(stub_responses: true, region: region)
resp = client.put_object(bucket:'name', key:'key', body:'data')
expect(resp.context.http_request.headers['authorization']).to match(
'AWS akid:')
end

it "forces v4 signing when aws:kms used for server side encryption" do
client = Client.new(stub_responses: true, region: region)
resp = client.put_object(
bucket: 'name',
key: 'key',
server_side_encryption: 'aws:kms',
body: 'data'
)
expect(resp.context.http_request.headers['authorization']).to match(
'AWS4-HMAC-SHA256')
end

end

[
"cn-north-1",
"eu-central-1",
"unknown-region",
Expand Down Expand Up @@ -165,14 +134,7 @@ module S3
end
end

it "defaults classic s3 signature us-east-1" do
client = Client.new(stub_responses: true, region: 'us-east-1')
resp = client.head_object(bucket:'name', key:'key')
expect(resp.context.http_request.headers['authorization']).to match(
'AWS akid:')
end

it "upgrades to signature version 4 when aws:kms used for sse" do
it "uses signature version 4 when aws:kms used for sse" do
client = Client.new(stub_responses: true, region: 'us-east-1')
resp = client.put_object(
bucket: 'name',
Expand Down