From 5569cafe2217c3c677956f416f9ee241d245fcd4 Mon Sep 17 00:00:00 2001
From: Eric Hanson <5846501+ericphanson@users.noreply.github.com>
Date: Wed, 11 May 2022 16:20:19 +0200
Subject: [PATCH 01/17] translate Retry to `Base.retry`

---
 .JuliaFormatter.toml               |  1 +
 Project.toml                       |  2 -
 src/AWS.jl                         |  1 -
 src/utilities/downloads_backend.jl | 23 ++++---
 src/utilities/request.jl           | 98 +++++++++++++++++-------------
 test/runtests.jl                   |  1 -
 6 files changed, 71 insertions(+), 55 deletions(-)
 create mode 100644 .JuliaFormatter.toml

diff --git a/.JuliaFormatter.toml b/.JuliaFormatter.toml
new file mode 100644
index 0000000000..1e72b507ee
--- /dev/null
+++ b/.JuliaFormatter.toml
@@ -0,0 +1 @@
+style="blue"
diff --git a/Project.toml b/Project.toml
index b42a527fa6..bbb8840658 100644
--- a/Project.toml
+++ b/Project.toml
@@ -15,7 +15,6 @@ JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 MbedTLS = "739be429-bea8-5141-9913-cc70e7f3736d"
 Mocking = "78c3b35d-d492-501b-9361-3d52fe80e533"
 OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
-Retry = "20febd7b-183b-5ae2-ac4a-720e7ce64774"
 Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
 URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
@@ -30,7 +29,6 @@ JSON = "0.18, 0.19, 0.20, 0.21"
 MbedTLS = "0.6, 0.7, 1"
 Mocking = "0.7"
 OrderedCollections = "1.3"
-Retry = "0.3, 0.4"
 URIs = "1"
 XMLDict = "0.3, 0.4"
 julia = "1.3"
diff --git a/src/AWS.jl b/src/AWS.jl
index 1b5508e583..fe09727a9f 100644
--- a/src/AWS.jl
+++ b/src/AWS.jl
@@ -8,7 +8,6 @@ using HTTP
 using MbedTLS
 using Mocking
 using OrderedCollections: LittleDict, OrderedDict
-using Retry
 using Sockets
 using URIs
 using UUIDs: UUIDs
diff --git a/src/utilities/downloads_backend.jl b/src/utilities/downloads_backend.jl
index ab31e4756b..540e4f8230 100644
--- a/src/utilities/downloads_backend.jl
+++ b/src/utilities/downloads_backend.jl
@@ -76,8 +76,9 @@ function _http_request(backend::DownloadsBackend, request::Request, response_str
 
     local buffer
     local response
-    try
-        @repeat 4 try
+
+    get_response =
+        () -> begin
             # Use a sacrificial I/O stream so that we only write the `response_stream` once
             # even with multiple attempts.
             buffer = Base.BufferStream()
@@ -100,13 +101,19 @@ function _http_request(backend::DownloadsBackend, request::Request, response_str
             )
 
             response = _http_response(request, r; throw=true)
-        catch e
-            @delay_retry if (
-                (isa(e, HTTP.StatusError) && AWS._http_status(e) >= 500) ||
-                isa(e, Downloads.RequestError)
-            )
-            end
+            # We'll rely on lexical scoping; `buffer` and `response`
+            # are bindings in the outer scope, so we don't need to return here.
+            return nothing
         end
+
+    check =
+        (s, e) -> begin
+            return (isa(e, HTTP.StatusError) && AWS._http_status(e) >= 500) ||
+                   isa(e, Downloads.RequestError)
+        end
+
+    try
+        retry(get_response; check=check, delays=Base.ExponentialBackOff(; n=4))
     finally
         close(buffer)
 
diff --git a/src/utilities/request.jl b/src/utilities/request.jl
index fe1a48c1fc..294bf3ecd0 100644
--- a/src/utilities/request.jl
+++ b/src/utilities/request.jl
@@ -116,57 +116,69 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
     request.headers["Host"] = HTTP.URI(request.url).host
     stream = @something request.response_stream IOBuffer()
 
-    @repeat 3 try
-        credentials(aws) === nothing || sign!(aws, request)
-
-        aws_response = @mock _http_request(request.backend, request, stream)
-        response = aws_response.response
-
-        if response.status in REDIRECT_ERROR_CODES
-            if HTTP.header(response, "Location") != ""
-                request.url = HTTP.header(response, "Location")
-            else
-                e = HTTP.StatusError(response.status, response)
-                throw(AWSException(e, stream))
+    local aws_response
+    local response
+
+    get_response =
+        () -> begin
+            credentials(aws) === nothing || sign!(aws, request)
+
+            aws_response = @mock _http_request(request.backend, request, stream)
+            response = aws_response.response
+
+            if response.status in REDIRECT_ERROR_CODES
+                if HTTP.header(response, "Location") != ""
+                    request.url = HTTP.header(response, "Location")
+                else
+                    e = HTTP.StatusError(response.status, response)
+                    throw(AWSException(e, stream))
+                end
             end
         end
-    catch e
-        if e isa HTTP.StatusError
-            e = AWSException(e, stream)
-        elseif !(e isa AWSException)
-            rethrow(e)
-        end
 
-        @retry if occursin("Signature expired", e.message)
-        end
+    check =
+        (s, e) -> begin
+            if e isa HTTP.StatusError
+                e = AWSException(e, stream)
+            elseif !(e isa AWSException)
+                rethrow(e)
+            end
 
-        # Handle ExpiredToken...
-        # https://github.com/aws/aws-sdk-go/blob/v1.31.5/aws/request/retryer.go#L98
-        @retry if e isa AWSException && e.code in EXPIRED_ERROR_CODES
-            check_credentials(credentials(aws); force_refresh=true)
-        end
+            occursin("Signature expired", e.message) && return true
 
-        # Throttle handling
-        # https://github.com/boto/botocore/blob/1.16.17/botocore/data/_retry.json
-        # https://docs.aws.amazon.com/general/latest/gr/api-retries.html
-        @delay_retry if _http_status(e.cause) == TOO_MANY_REQUESTS ||
-            e.code in THROTTLING_ERROR_CODES
-        end
+            # Handle ExpiredToken...
+            # https://github.com/aws/aws-sdk-go/blob/v1.31.5/aws/request/retryer.go#L98
+            if e isa AWSException && e.code in EXPIRED_ERROR_CODES
+                check_credentials(credentials(aws); force_refresh=true)
+                return true
+            end
 
-        # Handle BadDigest error and CRC32 check sum failure
-        @retry if _header(e.cause, "crc32body") == "x-amz-crc32" ||
-            e.code in ("BadDigest", "RequestTimeout", "RequestTimeoutException")
-        end
+            # Throttle handling
+            # https://github.com/boto/botocore/blob/1.16.17/botocore/data/_retry.json
+            # https://docs.aws.amazon.com/general/latest/gr/api-retries.html
+            if _http_status(e.cause) == TOO_MANY_REQUESTS ||
+                e.code in THROTTLING_ERROR_CODES
+                return true
+            end
 
-        if occursin("Missing Authentication Token", e.message) &&
-            aws.credentials === nothing
-            return throw(
-                NoCredentials(
-                    "You're attempting to perform a request without credentials set."
-                ),
-            )
+            # Handle BadDigest error and CRC32 check sum failure
+            if _header(e.cause, "crc32body") == "x-amz-crc32" ||
+                e.code in ("BadDigest", "RequestTimeout", "RequestTimeoutException")
+                return true
+            end
+
+            if occursin("Missing Authentication Token", e.message) &&
+                aws.credentials === nothing
+                return throw(
+                    NoCredentials(
+                        "You're attempting to perform a request without credentials set.",
+                    ),
+                )
+            end
+            return false
         end
-    end
+
+    retry(get_response; delays=Base.ExponentialBackOff(; n=3), check=check)
 
     if request.use_response_type
         return aws_response
diff --git a/test/runtests.jl b/test/runtests.jl
index e671e9939b..e4715d2ced 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -30,7 +30,6 @@ using MbedTLS: digest, MD_SHA256, MD_MD5
 using Mocking
 using Pkg
 using Random
-using Retry
 using Suppressor
 using Test
 using UUIDs

From 5b3f86b9b4c6de25c4e77607282d5737a4514794 Mon Sep 17 00:00:00 2001
From: Eric Hanson <5846501+ericphanson@users.noreply.github.com>
Date: Wed, 11 May 2022 16:45:17 +0200
Subject: [PATCH 02/17] update one more retry

---
 src/utilities/request.jl | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/src/utilities/request.jl b/src/utilities/request.jl
index 294bf3ecd0..b0bf041c5d 100644
--- a/src/utilities/request.jl
+++ b/src/utilities/request.jl
@@ -197,8 +197,9 @@ function _http_request(http_backend::HTTPBackend, request::Request, response_str
 
     local buffer
     local response
-    try
-        @repeat 4 try
+
+    get_response =
+        () -> begin
             # Use a sacrificial I/O stream so that we only write to the `response_stream`
             # once even with multiple attempted requests. Additionally this works around the
             # HTTP.jl issue (https://github.com/JuliaWeb/HTTP.jl/issues/543) where the
@@ -218,17 +219,28 @@ function _http_request(http_backend::HTTPBackend, request::Request, response_str
                 response_stream=buffer,
                 http_options...,
             )
-        catch e
+
+            # We'll rely on lexical scoping; `buffer` and `response`
+            # are bindings in the outer scope, so we don't need to return here.
+            return nothing
+        end
+
+    check =
+        (s, e) -> begin
             # `Base.IOError` is needed because HTTP.jl can often have errors that aren't
             # caught and wrapped in an `HTTP.IOError`
             # https://github.com/JuliaWeb/HTTP.jl/issues/382
-            @delay_retry if isa(e, Sockets.DNSError) ||
+            if isa(e, Sockets.DNSError) ||
                 isa(e, HTTP.ParseError) ||
                 isa(e, HTTP.IOError) ||
                 isa(e, Base.IOError) ||
                 (isa(e, HTTP.StatusError) && _http_status(e) >= 500)
+                return true
             end
         end
+
+    try
+        retry(get_response; check=check, delays=Base.ExponentialBackOff(; n=4))
     finally
         # We're unable to read from the `Base.BufferStream` until it has been closed.
         # HTTP.jl will close passed in `response_stream` keyword. This ensures that it

From 3c129fac4d9c5db1befdb905827907984cf5e674 Mon Sep 17 00:00:00 2001
From: Eric Hanson <5846501+ericphanson@users.noreply.github.com>
Date: Wed, 11 May 2022 16:54:58 +0200
Subject: [PATCH 03/17] fix logic

---
 src/utilities/request.jl | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/src/utilities/request.jl b/src/utilities/request.jl
index b0bf041c5d..2d48a4bbb7 100644
--- a/src/utilities/request.jl
+++ b/src/utilities/request.jl
@@ -230,13 +230,11 @@ function _http_request(http_backend::HTTPBackend, request::Request, response_str
             # `Base.IOError` is needed because HTTP.jl can often have errors that aren't
             # caught and wrapped in an `HTTP.IOError`
             # https://github.com/JuliaWeb/HTTP.jl/issues/382
-            if isa(e, Sockets.DNSError) ||
-                isa(e, HTTP.ParseError) ||
-                isa(e, HTTP.IOError) ||
-                isa(e, Base.IOError) ||
-                (isa(e, HTTP.StatusError) && _http_status(e) >= 500)
-                return true
-            end
+            return isa(e, Sockets.DNSError) ||
+                   isa(e, HTTP.ParseError) ||
+                   isa(e, HTTP.IOError) ||
+                   isa(e, Base.IOError) ||
+                   (isa(e, HTTP.StatusError) && _http_status(e) >= 500)
         end
 
     try

From 00f24ec040baba78a5765cac4198c956b2299001 Mon Sep 17 00:00:00 2001
From: Eric Hanson <5846501+ericphanson@users.noreply.github.com>
Date: Wed, 11 May 2022 17:06:00 +0200
Subject: [PATCH 04/17] actually call the function

---
 src/utilities/downloads_backend.jl | 5 ++++-
 src/utilities/request.jl           | 8 ++++++--
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/utilities/downloads_backend.jl b/src/utilities/downloads_backend.jl
index 540e4f8230..5053a5927a 100644
--- a/src/utilities/downloads_backend.jl
+++ b/src/utilities/downloads_backend.jl
@@ -112,8 +112,11 @@ function _http_request(backend::DownloadsBackend, request::Request, response_str
                    isa(e, Downloads.RequestError)
         end
 
+    get_response_with_retry = retry(
+        get_response; check=check, delays=Base.ExponentialBackOff(; n=4)
+    )
     try
-        retry(get_response; check=check, delays=Base.ExponentialBackOff(; n=4))
+        get_response_with_retry()
     finally
         close(buffer)
 
diff --git a/src/utilities/request.jl b/src/utilities/request.jl
index 2d48a4bbb7..e203b6ce64 100644
--- a/src/utilities/request.jl
+++ b/src/utilities/request.jl
@@ -178,7 +178,7 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
             return false
         end
 
-    retry(get_response; delays=Base.ExponentialBackOff(; n=3), check=check)
+    retry(get_response; delays=Base.ExponentialBackOff(; n=3), check=check)()
 
     if request.use_response_type
         return aws_response
@@ -237,8 +237,12 @@ function _http_request(http_backend::HTTPBackend, request::Request, response_str
                    (isa(e, HTTP.StatusError) && _http_status(e) >= 500)
         end
 
+    get_response_with_retry = retry(
+        get_response; check=check, delays=Base.ExponentialBackOff(; n=4)
+    )
+
     try
-        retry(get_response; check=check, delays=Base.ExponentialBackOff(; n=4))
+        get_response_with_retry()
     finally
         # We're unable to read from the `Base.BufferStream` until it has been closed.
         # HTTP.jl will close passed in `response_stream` keyword. This ensures that it

From f226445420da9dd2037b58e6f2eb17813cbd0554 Mon Sep 17 00:00:00 2001
From: Eric Hanson <5846501+ericphanson@users.noreply.github.com>
Date: Wed, 11 May 2022 19:30:43 +0200
Subject: [PATCH 05/17] try to fix things

---
 src/utilities/request.jl | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/utilities/request.jl b/src/utilities/request.jl
index e203b6ce64..9b302ba5d8 100644
--- a/src/utilities/request.jl
+++ b/src/utilities/request.jl
@@ -136,14 +136,20 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
             end
         end
 
-    check =
-        (s, e) -> begin
+    function upgrade_error(f)
+        () -> try
+            return f()
+        catch e
             if e isa HTTP.StatusError
                 e = AWSException(e, stream)
-            elseif !(e isa AWSException)
                 rethrow(e)
             end
+            rethrow()
+        end
+    end
 
+    check =
+        (s, e) -> begin
             occursin("Signature expired", e.message) && return true
 
             # Handle ExpiredToken...
@@ -178,7 +184,7 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
             return false
         end
 
-    retry(get_response; delays=Base.ExponentialBackOff(; n=3), check=check)()
+    retry(upgrade_error(get_response); delays=Base.ExponentialBackOff(; n=3), check=check)()
 
     if request.use_response_type
         return aws_response

From e91a4f1e92c06fbb129f96dd67fd0ba03332889a Mon Sep 17 00:00:00 2001
From: Eric Hanson <5846501+ericphanson@users.noreply.github.com>
Date: Wed, 11 May 2022 19:38:40 +0200
Subject: [PATCH 06/17] match number of retries properly

---
 src/utilities/downloads_backend.jl | 2 +-
 src/utilities/request.jl           | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/utilities/downloads_backend.jl b/src/utilities/downloads_backend.jl
index 5053a5927a..eda9c4c95e 100644
--- a/src/utilities/downloads_backend.jl
+++ b/src/utilities/downloads_backend.jl
@@ -113,7 +113,7 @@ function _http_request(backend::DownloadsBackend, request::Request, response_str
         end
 
     get_response_with_retry = retry(
-        get_response; check=check, delays=Base.ExponentialBackOff(; n=4)
+        get_response; check=check, delays=Base.ExponentialBackOff(; n=3)
     )
     try
         get_response_with_retry()
diff --git a/src/utilities/request.jl b/src/utilities/request.jl
index 9b302ba5d8..5523a9a3b3 100644
--- a/src/utilities/request.jl
+++ b/src/utilities/request.jl
@@ -184,7 +184,7 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
             return false
         end
 
-    retry(upgrade_error(get_response); delays=Base.ExponentialBackOff(; n=3), check=check)()
+    retry(upgrade_error(get_response); delays=Base.ExponentialBackOff(; n=2), check=check)()
 
     if request.use_response_type
         return aws_response
@@ -244,7 +244,7 @@ function _http_request(http_backend::HTTPBackend, request::Request, response_str
         end
 
     get_response_with_retry = retry(
-        get_response; check=check, delays=Base.ExponentialBackOff(; n=4)
+        get_response; check=check, delays=Base.ExponentialBackOff(; n=3)
     )
 
     try

From 8616f1e7cdeb8a7bc20c296ab87bb9d22fa6a084 Mon Sep 17 00:00:00 2001
From: Eric Hanson <5846501+ericphanson@users.noreply.github.com>
Date: Wed, 11 May 2022 19:54:35 +0200
Subject: [PATCH 07/17] fix

---
 src/utilities/request.jl | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/utilities/request.jl b/src/utilities/request.jl
index 5523a9a3b3..f5abf23972 100644
--- a/src/utilities/request.jl
+++ b/src/utilities/request.jl
@@ -150,6 +150,11 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
 
     check =
         (s, e) -> begin
+            # Pass on non-AWS exceptions.
+            if !(e isa AWSException)
+                return false
+            end
+
             occursin("Signature expired", e.message) && return true
 
             # Handle ExpiredToken...

From cf9c55a52837a6c0ece50a8e3167b131626d234f Mon Sep 17 00:00:00 2001
From: Eric Hanson <5846501+ericphanson@users.noreply.github.com>
Date: Thu, 12 May 2022 16:45:46 +0200
Subject: [PATCH 08/17] add and use `AWSExponentialBackoff`

---
 src/utilities/downloads_backend.jl |  2 +-
 src/utilities/request.jl           |  8 ++++++--
 src/utilities/utilities.jl         | 23 +++++++++++++++++++++++
 test/utilities.jl                  | 22 ++++++++++++++++++++++
 4 files changed, 52 insertions(+), 3 deletions(-)

diff --git a/src/utilities/downloads_backend.jl b/src/utilities/downloads_backend.jl
index eda9c4c95e..88e1b87719 100644
--- a/src/utilities/downloads_backend.jl
+++ b/src/utilities/downloads_backend.jl
@@ -113,7 +113,7 @@ function _http_request(backend::DownloadsBackend, request::Request, response_str
         end
 
     get_response_with_retry = retry(
-        get_response; check=check, delays=Base.ExponentialBackOff(; n=3)
+        get_response; check=check, delays=AWSExponentialBackoff(; max_attempts=4)
     )
     try
         get_response_with_retry()
diff --git a/src/utilities/request.jl b/src/utilities/request.jl
index f5abf23972..c51d7cf33d 100644
--- a/src/utilities/request.jl
+++ b/src/utilities/request.jl
@@ -189,7 +189,11 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
             return false
         end
 
-    retry(upgrade_error(get_response); delays=Base.ExponentialBackOff(; n=2), check=check)()
+    retry(
+        upgrade_error(get_response);
+        delays=AWSExponentialBackoff(; max_attempts=3),
+        check=check,
+    )()
 
     if request.use_response_type
         return aws_response
@@ -249,7 +253,7 @@ function _http_request(http_backend::HTTPBackend, request::Request, response_str
         end
 
     get_response_with_retry = retry(
-        get_response; check=check, delays=Base.ExponentialBackOff(; n=3)
+        get_response; check=check, delays=AWSExponentialBackoff(; max_attempts=4)
     )
 
     try
diff --git a/src/utilities/utilities.jl b/src/utilities/utilities.jl
index 427ef49bc0..cdc7ab2b2d 100644
--- a/src/utilities/utilities.jl
+++ b/src/utilities/utilities.jl
@@ -139,3 +139,26 @@ end
 function _assignment_to_kw!(x)
     return throw(ArgumentError("Expected assignment expression, instead found: `$x`"))
 end
+
+# https://docs.aws.amazon.com/sdkref/latest/guide/feature-retry-behavior.html
+struct AWSExponentialBackoff
+    max_attempts::Int
+    max_backoff::Float64
+end
+
+# Defaults for AWS's `standard` retry mode. Note: these can be overridden elsewhere.
+function AWSExponentialBackoff(; max_attempts=3, max_backoff=20.0)
+    return AWSExponentialBackoff(max_attempts, max_backoff)
+end
+
+# We make one more attempt than the number of delays
+Base.length(exp::AWSExponentialBackoff) = exp.max_attempts - 1
+
+function Base.iterate(exp::AWSExponentialBackoff, i=1)
+    i >= exp.max_attempts && return nothing
+    # rand() has values in [0, 1), so we use 1.0 - rand() which has values in (0, 1] required.
+    b = 1.0 - rand()
+    r = 2.0
+    delay = min(b * r^i, exp.max_backoff)
+    return delay, i + 1
+end
diff --git a/test/utilities.jl b/test/utilities.jl
index f444983d93..3edcb02f76 100644
--- a/test/utilities.jl
+++ b/test/utilities.jl
@@ -79,3 +79,25 @@ end
         @test ex == Expr(:kw, :a, true)
     end
 end
+
+# Are we iterating the right number of times?
+function count_len(itr)
+    c = 0
+    result = iterate(itr)
+    while result !== nothing
+        c += 1
+        state = result[2]
+        result = iterate(itr, state)
+    end
+    return c
+end
+
+@testset "AWSExponentialBackoff" begin
+    for (n, max_backoff) in [(3, 5.0), (10, 20.0)]
+        itr = AWS.AWSExponentialBackoff(; max_attempts=n, max_backoff=max_backoff)
+        @test count_len(itr) == n - 1
+        @test length(collect(itr)) == n - 1
+        @test all(>(0), itr)
+        @test all(<=(max_backoff), itr)
+    end
+end

From 058a51d7adf6006277080544a9540c44a84a491b Mon Sep 17 00:00:00 2001
From: Eric Hanson <5846501+ericphanson@users.noreply.github.com>
Date: Tue, 17 May 2022 22:02:30 +0200
Subject: [PATCH 09/17] Apply suggestions from code review

Co-authored-by: Curtis Vogt <curtis.vogt@gmail.com>
---
 src/utilities/downloads_backend.jl |  7 +++----
 src/utilities/request.jl           | 29 +++++++++++++++--------------
 src/utilities/utilities.jl         | 14 +++++---------
 3 files changed, 23 insertions(+), 27 deletions(-)

diff --git a/src/utilities/downloads_backend.jl b/src/utilities/downloads_backend.jl
index 88e1b87719..5058988c63 100644
--- a/src/utilities/downloads_backend.jl
+++ b/src/utilities/downloads_backend.jl
@@ -77,8 +77,8 @@ function _http_request(backend::DownloadsBackend, request::Request, response_str
     local buffer
     local response
 
-    get_response =
-        () -> begin
+    try
+        retry(; check, delays) do
             # Use a sacrificial I/O stream so that we only write the `response_stream` once
             # even with multiple attempts.
             buffer = Base.BufferStream()
@@ -106,8 +106,7 @@ function _http_request(backend::DownloadsBackend, request::Request, response_str
             return nothing
         end
 
-    check =
-        (s, e) -> begin
+    check = function (s, e)
             return (isa(e, HTTP.StatusError) && AWS._http_status(e) >= 500) ||
                    isa(e, Downloads.RequestError)
         end
diff --git a/src/utilities/request.jl b/src/utilities/request.jl
index c51d7cf33d..0f9f463e6c 100644
--- a/src/utilities/request.jl
+++ b/src/utilities/request.jl
@@ -148,8 +148,7 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
         end
     end
 
-    check =
-        (s, e) -> begin
+    check = function (s, e)
             # Pass on non-AWS exceptions.
             if !(e isa AWSException)
                 return false
@@ -189,11 +188,14 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
             return false
         end
 
-    retry(
-        upgrade_error(get_response);
-        delays=AWSExponentialBackoff(; max_attempts=3),
-        check=check,
-    )()
+    delays = AWSExponentialBackoff(; max_attempts=3)
+    
+    retry(; check, delays) do
+        # Inline both of these functions
+        upgrade_error() do
+            get_response()
+        end
+    end
 
     if request.use_response_type
         return aws_response
@@ -240,8 +242,7 @@ function _http_request(http_backend::HTTPBackend, request::Request, response_str
             return nothing
         end
 
-    check =
-        (s, e) -> begin
+    check = function (s, e)
             # `Base.IOError` is needed because HTTP.jl can often have errors that aren't
             # caught and wrapped in an `HTTP.IOError`
             # https://github.com/JuliaWeb/HTTP.jl/issues/382
@@ -252,12 +253,12 @@ function _http_request(http_backend::HTTPBackend, request::Request, response_str
                    (isa(e, HTTP.StatusError) && _http_status(e) >= 500)
         end
 
-    get_response_with_retry = retry(
-        get_response; check=check, delays=AWSExponentialBackoff(; max_attempts=4)
-    )
-
+    delays=AWSExponentialBackoff(; max_attempts=4)
+    
     try
-        get_response_with_retry()
+        retry(; check, delays) do
+            get_response()   # Inline this function
+        end
     finally
         # We're unable to read from the `Base.BufferStream` until it has been closed.
         # HTTP.jl will close passed in `response_stream` keyword. This ensures that it
diff --git a/src/utilities/utilities.jl b/src/utilities/utilities.jl
index cdc7ab2b2d..7c8d8bd2c3 100644
--- a/src/utilities/utilities.jl
+++ b/src/utilities/utilities.jl
@@ -141,14 +141,10 @@ function _assignment_to_kw!(x)
 end
 
 # https://docs.aws.amazon.com/sdkref/latest/guide/feature-retry-behavior.html
-struct AWSExponentialBackoff
-    max_attempts::Int
-    max_backoff::Float64
-end
-
-# Defaults for AWS's `standard` retry mode. Note: these can be overridden elsewhere.
-function AWSExponentialBackoff(; max_attempts=3, max_backoff=20.0)
-    return AWSExponentialBackoff(max_attempts, max_backoff)
+# Default values for AWS's `standard` retry mode. Note: these can be overridden elsewhere.
+Base.@kwdef struct AWSExponentialBackoff
+    max_attempts::Int = 3
+    max_backoff::Float64 = 20.0
 end
 
 # We make one more attempt than the number of delays
@@ -157,7 +153,7 @@ Base.length(exp::AWSExponentialBackoff) = exp.max_attempts - 1
 function Base.iterate(exp::AWSExponentialBackoff, i=1)
     i >= exp.max_attempts && return nothing
     # rand() has values in [0, 1), so we use 1.0 - rand() which has values in (0, 1] required.
-    b = 1.0 - rand()
+    b = 1.0 - rand(exp.rng)
     r = 2.0
     delay = min(b * r^i, exp.max_backoff)
     return delay, i + 1

From b6a61da0840d56a3e7000196eee854a5c36ed593 Mon Sep 17 00:00:00 2001
From: Eric Hanson <5846501+ericphanson@users.noreply.github.com>
Date: Tue, 17 May 2022 22:33:56 +0200
Subject: [PATCH 10/17] formatting fixes; random seeding

---
 Project.toml                       |   5 +-
 src/AWS.jl                         |   3 +-
 src/utilities/downloads_backend.jl |  70 +++++++-------
 src/utilities/request.jl           | 149 ++++++++++++++---------------
 src/utilities/utilities.jl         |   1 +
 test/runtests.jl                   |   1 +
 test/utilities.jl                  |   2 +-
 7 files changed, 113 insertions(+), 118 deletions(-)

diff --git a/Project.toml b/Project.toml
index bbb8840658..7eb820fd41 100644
--- a/Project.toml
+++ b/Project.toml
@@ -15,6 +15,7 @@ JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 MbedTLS = "739be429-bea8-5141-9913-cc70e7f3736d"
 Mocking = "78c3b35d-d492-501b-9361-3d52fe80e533"
 OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
 URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
@@ -29,6 +30,7 @@ JSON = "0.18, 0.19, 0.20, 0.21"
 MbedTLS = "0.6, 0.7, 1"
 Mocking = "0.7"
 OrderedCollections = "1.3"
+StableRNGs = "1"
 URIs = "1"
 XMLDict = "0.3, 0.4"
 julia = "1.3"
@@ -37,8 +39,9 @@ julia = "1.3"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb"
+StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [targets]
-test = ["Pkg", "Random", "Suppressor", "Test", "UUIDs"]
+test = ["Pkg", "Random", "Suppressor", "StableRNGs", "Test", "UUIDs"]
diff --git a/src/AWS.jl b/src/AWS.jl
index fe09727a9f..88506aeb6f 100644
--- a/src/AWS.jl
+++ b/src/AWS.jl
@@ -1,6 +1,6 @@
 module AWS
 
-using Compat: Compat, @something
+using Compat: Compat, @something, @compat
 using Base64
 using Dates
 using Downloads: Downloads, Downloader, Curl
@@ -12,6 +12,7 @@ using Sockets
 using URIs
 using UUIDs: UUIDs
 using XMLDict
+using Random
 
 export @service
 export _merge
diff --git a/src/utilities/downloads_backend.jl b/src/utilities/downloads_backend.jl
index 5058988c63..c432f7497c 100644
--- a/src/utilities/downloads_backend.jl
+++ b/src/utilities/downloads_backend.jl
@@ -77,45 +77,43 @@ function _http_request(backend::DownloadsBackend, request::Request, response_str
     local buffer
     local response
 
-    try
-        retry(; check, delays) do
-            # Use a sacrificial I/O stream so that we only write the `response_stream` once
-            # even with multiple attempts.
-            buffer = Base.BufferStream()
-
-            # Rewind the input on each attempt otherwise every subsequent attempt will send an
-            # empty payload.
-            input !== nothing && seekstart(input)
-
-            r = @mock Downloads.request(
-                request.url;
-                input=input,
-                # Compatibility with Downloads.jl versions below v1.5.2
-                # See: https://github.com/JuliaLang/Downloads.jl/issues/131
-                output=request.request_method != "HEAD" ? buffer : nothing,
-                method=request.request_method,
-                headers=request.headers,
-                verbose=false,
-                throw=true,
-                downloader=downloader,
-            )
-
-            response = _http_response(request, r; throw=true)
-            # We'll rely on lexical scoping; `buffer` and `response`
-            # are bindings in the outer scope, so we don't need to return here.
-            return nothing
-        end
-
     check = function (s, e)
-            return (isa(e, HTTP.StatusError) && AWS._http_status(e) >= 500) ||
-                   isa(e, Downloads.RequestError)
-        end
+        return (isa(e, HTTP.StatusError) && AWS._http_status(e) >= 500) ||
+               isa(e, Downloads.RequestError)
+    end
+
+    delays = AWSExponentialBackoff(; max_attempts=4)
+
+    get_response = function ()
+        # Use a sacrificial I/O stream so that we only write the `response_stream` once
+        # even with multiple attempts.
+        buffer = Base.BufferStream()
+
+        # Rewind the input on each attempt otherwise every subsequent attempt will send an
+        # empty payload.
+        input !== nothing && seekstart(input)
+
+        r = @mock Downloads.request(
+            request.url;
+            input=input,
+            # Compatibility with Downloads.jl versions below v1.5.2
+            # See: https://github.com/JuliaLang/Downloads.jl/issues/131
+            output=request.request_method != "HEAD" ? buffer : nothing,
+            method=request.request_method,
+            headers=request.headers,
+            verbose=false,
+            throw=true,
+            downloader=downloader,
+        )
+
+        response = _http_response(request, r; throw=true)
+        # We'll rely on lexical scoping; `buffer` and `response`
+        # are bindings in the outer scope, so we don't need to return here.
+        return nothing
+    end
 
-    get_response_with_retry = retry(
-        get_response; check=check, delays=AWSExponentialBackoff(; max_attempts=4)
-    )
     try
-        get_response_with_retry()
+        @compat retry(get_response; check, delays)()
     finally
         close(buffer)
 
diff --git a/src/utilities/request.jl b/src/utilities/request.jl
index 0f9f463e6c..005bd0dd7c 100644
--- a/src/utilities/request.jl
+++ b/src/utilities/request.jl
@@ -149,54 +149,48 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
     end
 
     check = function (s, e)
-            # Pass on non-AWS exceptions.
-            if !(e isa AWSException)
-                return false
-            end
+        # Pass on non-AWS exceptions.
+        if !(e isa AWSException)
+            return false
+        end
 
-            occursin("Signature expired", e.message) && return true
+        occursin("Signature expired", e.message) && return true
 
-            # Handle ExpiredToken...
-            # https://github.com/aws/aws-sdk-go/blob/v1.31.5/aws/request/retryer.go#L98
-            if e isa AWSException && e.code in EXPIRED_ERROR_CODES
-                check_credentials(credentials(aws); force_refresh=true)
-                return true
-            end
-
-            # Throttle handling
-            # https://github.com/boto/botocore/blob/1.16.17/botocore/data/_retry.json
-            # https://docs.aws.amazon.com/general/latest/gr/api-retries.html
-            if _http_status(e.cause) == TOO_MANY_REQUESTS ||
-                e.code in THROTTLING_ERROR_CODES
-                return true
-            end
+        # Handle ExpiredToken...
+        # https://github.com/aws/aws-sdk-go/blob/v1.31.5/aws/request/retryer.go#L98
+        if e isa AWSException && e.code in EXPIRED_ERROR_CODES
+            check_credentials(credentials(aws); force_refresh=true)
+            return true
+        end
 
-            # Handle BadDigest error and CRC32 check sum failure
-            if _header(e.cause, "crc32body") == "x-amz-crc32" ||
-                e.code in ("BadDigest", "RequestTimeout", "RequestTimeoutException")
-                return true
-            end
+        # Throttle handling
+        # https://github.com/boto/botocore/blob/1.16.17/botocore/data/_retry.json
+        # https://docs.aws.amazon.com/general/latest/gr/api-retries.html
+        if _http_status(e.cause) == TOO_MANY_REQUESTS || e.code in THROTTLING_ERROR_CODES
+            return true
+        end
 
-            if occursin("Missing Authentication Token", e.message) &&
-                aws.credentials === nothing
-                return throw(
-                    NoCredentials(
-                        "You're attempting to perform a request without credentials set.",
-                    ),
-                )
-            end
-            return false
+        # Handle BadDigest error and CRC32 check sum failure
+        if _header(e.cause, "crc32body") == "x-amz-crc32" ||
+            e.code in ("BadDigest", "RequestTimeout", "RequestTimeoutException")
+            return true
         end
 
-    delays = AWSExponentialBackoff(; max_attempts=3)
-    
-    retry(; check, delays) do
-        # Inline both of these functions
-        upgrade_error() do
-            get_response()
+        if occursin("Missing Authentication Token", e.message) &&
+            aws.credentials === nothing
+            return throw(
+                NoCredentials(
+                    "You're attempting to perform a request without credentials set."
+                ),
+            )
         end
+        return false
     end
 
+    delays = AWSExponentialBackoff(; max_attempts=3)
+
+    @compat retry(upgrade_error(get_response); check, delays)()
+
     if request.use_response_type
         return aws_response
     else
@@ -215,50 +209,47 @@ function _http_request(http_backend::HTTPBackend, request::Request, response_str
     local buffer
     local response
 
-    get_response =
-        () -> begin
-            # Use a sacrificial I/O stream so that we only write to the `response_stream`
-            # once even with multiple attempted requests. Additionally this works around the
-            # HTTP.jl issue (https://github.com/JuliaWeb/HTTP.jl/issues/543) where the
-            # `response_stream` is closed automatically. Effectively, this works as if we're
-            # not using streaming I/O at all, as we write all data at once, but only
-            # returning data via I/O ensures we aren't relying on response's body being
-            # populated.
-            buffer = Base.BufferStream()
-
-            response = @mock HTTP.request(
-                http_stack,
-                request.request_method,
-                HTTP.URI(request.url),
-                HTTP.mkheaders(request.headers),
-                request.content;
-                require_ssl_verification=false,
-                response_stream=buffer,
-                http_options...,
-            )
-
-            # We'll rely on lexical scoping; `buffer` and `response`
-            # are bindings in the outer scope, so we don't need to return here.
-            return nothing
-        end
+    get_response = function ()
+        # Use a sacrificial I/O stream so that we only write to the `response_stream`
+        # once even with multiple attempted requests. Additionally this works around the
+        # HTTP.jl issue (https://github.com/JuliaWeb/HTTP.jl/issues/543) where the
+        # `response_stream` is closed automatically. Effectively, this works as if we're
+        # not using streaming I/O at all, as we write all data at once, but only
+        # returning data via I/O ensures we aren't relying on response's body being
+        # populated.
+        buffer = Base.BufferStream()
+
+        response = @mock HTTP.request(
+            http_stack,
+            request.request_method,
+            HTTP.URI(request.url),
+            HTTP.mkheaders(request.headers),
+            request.content;
+            require_ssl_verification=false,
+            response_stream=buffer,
+            http_options...,
+        )
+
+        # We'll rely on lexical scoping; `buffer` and `response`
+        # are bindings in the outer scope, so we don't need to return here.
+        return nothing
+    end
 
     check = function (s, e)
-            # `Base.IOError` is needed because HTTP.jl can often have errors that aren't
-            # caught and wrapped in an `HTTP.IOError`
-            # https://github.com/JuliaWeb/HTTP.jl/issues/382
-            return isa(e, Sockets.DNSError) ||
-                   isa(e, HTTP.ParseError) ||
-                   isa(e, HTTP.IOError) ||
-                   isa(e, Base.IOError) ||
-                   (isa(e, HTTP.StatusError) && _http_status(e) >= 500)
-        end
+        # `Base.IOError` is needed because HTTP.jl can often have errors that aren't
+        # caught and wrapped in an `HTTP.IOError`
+        # https://github.com/JuliaWeb/HTTP.jl/issues/382
+        return isa(e, Sockets.DNSError) ||
+               isa(e, HTTP.ParseError) ||
+               isa(e, HTTP.IOError) ||
+               isa(e, Base.IOError) ||
+               (isa(e, HTTP.StatusError) && _http_status(e) >= 500)
+    end
+
+    delays = AWSExponentialBackoff(; max_attempts=4)
 
-    delays=AWSExponentialBackoff(; max_attempts=4)
-    
     try
-        retry(; check, delays) do
-            get_response()   # Inline this function
-        end
+        @compat retry(get_response; check, delays)()
     finally
         # We're unable to read from the `Base.BufferStream` until it has been closed.
         # HTTP.jl will close passed in `response_stream` keyword. This ensures that it
diff --git a/src/utilities/utilities.jl b/src/utilities/utilities.jl
index 7c8d8bd2c3..52eaca2518 100644
--- a/src/utilities/utilities.jl
+++ b/src/utilities/utilities.jl
@@ -145,6 +145,7 @@ end
 Base.@kwdef struct AWSExponentialBackoff
     max_attempts::Int = 3
     max_backoff::Float64 = 20.0
+    rng::AbstractRNG = Random.GLOBAL_RNG
 end
 
 # We make one more attempt than the number of delays
diff --git a/test/runtests.jl b/test/runtests.jl
index e4715d2ced..3df8afe121 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -34,6 +34,7 @@ using Suppressor
 using Test
 using UUIDs
 using XMLDict
+using StableRNGs
 
 Mocking.activate()
 
diff --git a/test/utilities.jl b/test/utilities.jl
index 3edcb02f76..f5f76bb83c 100644
--- a/test/utilities.jl
+++ b/test/utilities.jl
@@ -94,7 +94,7 @@ end
 
 @testset "AWSExponentialBackoff" begin
     for (n, max_backoff) in [(3, 5.0), (10, 20.0)]
-        itr = AWS.AWSExponentialBackoff(; max_attempts=n, max_backoff=max_backoff)
+        itr = AWS.AWSExponentialBackoff(; max_attempts=n, max_backoff=max_backoff, rng=StableRNG(1))
         @test count_len(itr) == n - 1
         @test length(collect(itr)) == n - 1
         @test all(>(0), itr)

From 64f8a4ab694cff47981babd0ea82f9c9491ceb22 Mon Sep 17 00:00:00 2001
From: Eric Hanson <5846501+ericphanson@users.noreply.github.com>
Date: Tue, 17 May 2022 22:49:20 +0200
Subject: [PATCH 11/17] Update test/utilities.jl

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
---
 test/utilities.jl | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/utilities.jl b/test/utilities.jl
index f5f76bb83c..b14dc342b0 100644
--- a/test/utilities.jl
+++ b/test/utilities.jl
@@ -94,7 +94,9 @@ end
 
 @testset "AWSExponentialBackoff" begin
     for (n, max_backoff) in [(3, 5.0), (10, 20.0)]
-        itr = AWS.AWSExponentialBackoff(; max_attempts=n, max_backoff=max_backoff, rng=StableRNG(1))
+        itr = AWS.AWSExponentialBackoff(;
+            max_attempts=n, max_backoff=max_backoff, rng=StableRNG(1)
+        )
         @test count_len(itr) == n - 1
         @test length(collect(itr)) == n - 1
         @test all(>(0), itr)

From ee14b9fffa309c29233aea18e6f77f215cb5f95c Mon Sep 17 00:00:00 2001
From: Eric Hanson <5846501+ericphanson@users.noreply.github.com>
Date: Thu, 19 May 2022 15:09:52 +0200
Subject: [PATCH 12/17] don't use `@compat`; not working on 1.3

---
 src/AWS.jl                         | 2 +-
 src/utilities/downloads_backend.jl | 2 +-
 src/utilities/request.jl           | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/AWS.jl b/src/AWS.jl
index 88506aeb6f..78d35d751f 100644
--- a/src/AWS.jl
+++ b/src/AWS.jl
@@ -1,6 +1,6 @@
 module AWS
 
-using Compat: Compat, @something, @compat
+using Compat: Compat, @something
 using Base64
 using Dates
 using Downloads: Downloads, Downloader, Curl
diff --git a/src/utilities/downloads_backend.jl b/src/utilities/downloads_backend.jl
index c432f7497c..00c298ad06 100644
--- a/src/utilities/downloads_backend.jl
+++ b/src/utilities/downloads_backend.jl
@@ -113,7 +113,7 @@ function _http_request(backend::DownloadsBackend, request::Request, response_str
     end
 
     try
-        @compat retry(get_response; check, delays)()
+        retry(get_response; check=check, delays=delays)()
     finally
         close(buffer)
 
diff --git a/src/utilities/request.jl b/src/utilities/request.jl
index 005bd0dd7c..f3bea64a36 100644
--- a/src/utilities/request.jl
+++ b/src/utilities/request.jl
@@ -189,7 +189,7 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
 
     delays = AWSExponentialBackoff(; max_attempts=3)
 
-    @compat retry(upgrade_error(get_response); check, delays)()
+    retry(upgrade_error(get_response); check=check, delays=delays)()
 
     if request.use_response_type
         return aws_response
@@ -249,7 +249,7 @@ function _http_request(http_backend::HTTPBackend, request::Request, response_str
     delays = AWSExponentialBackoff(; max_attempts=4)
 
     try
-        @compat retry(get_response; check, delays)()
+        retry(get_response; check=check, delays=delays)()
     finally
         # We're unable to read from the `Base.BufferStream` until it has been closed.
         # HTTP.jl will close passed in `response_stream` keyword. This ensures that it

From c841f52656ca40fa15a5f7b9e6e03f4d2faa7d1c Mon Sep 17 00:00:00 2001
From: Eric Hanson <5846501+ericphanson@users.noreply.github.com>
Date: Thu, 19 May 2022 17:17:56 +0200
Subject: [PATCH 13/17] Apply suggestions from code review

Co-authored-by: Curtis Vogt <curtis.vogt@gmail.com>
---
 Project.toml             |  3 +--
 src/utilities/request.jl | 31 +++++++++++++++----------------
 2 files changed, 16 insertions(+), 18 deletions(-)

diff --git a/Project.toml b/Project.toml
index 7eb820fd41..1cd86fa4a8 100644
--- a/Project.toml
+++ b/Project.toml
@@ -37,11 +37,10 @@ julia = "1.3"
 
 [extras]
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
-Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb"
 StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [targets]
-test = ["Pkg", "Random", "Suppressor", "StableRNGs", "Test", "UUIDs"]
+test = ["Pkg", "Suppressor", "StableRNGs", "Test", "UUIDs"]
diff --git a/src/utilities/request.jl b/src/utilities/request.jl
index f3bea64a36..9615490c83 100644
--- a/src/utilities/request.jl
+++ b/src/utilities/request.jl
@@ -119,25 +119,24 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
     local aws_response
     local response
 
-    get_response =
-        () -> begin
-            credentials(aws) === nothing || sign!(aws, request)
-
-            aws_response = @mock _http_request(request.backend, request, stream)
-            response = aws_response.response
-
-            if response.status in REDIRECT_ERROR_CODES
-                if HTTP.header(response, "Location") != ""
-                    request.url = HTTP.header(response, "Location")
-                else
-                    e = HTTP.StatusError(response.status, response)
-                    throw(AWSException(e, stream))
-                end
+    get_response = function ()
+        credentials(aws) === nothing || sign!(aws, request)
+
+        aws_response = @mock _http_request(request.backend, request, stream)
+        response = aws_response.response
+
+        if response.status in REDIRECT_ERROR_CODES
+            if HTTP.header(response, "Location") != ""
+                request.url = HTTP.header(response, "Location")
+            else
+                e = HTTP.StatusError(response.status, response)
+                throw(AWSException(e, stream))
             end
         end
+    end
 
     function upgrade_error(f)
-        () -> try
+        try
             return f()
         catch e
             if e isa HTTP.StatusError
@@ -189,7 +188,7 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
 
     delays = AWSExponentialBackoff(; max_attempts=3)
 
-    retry(upgrade_error(get_response); check=check, delays=delays)()
+    retry(upgrade_error ∘ get_response; check=check, delays=delays)()
 
     if request.use_response_type
         return aws_response

From 64a6494a3456777516cbc1aa83c2171526a83fb1 Mon Sep 17 00:00:00 2001
From: Eric Hanson <5846501+ericphanson@users.noreply.github.com>
Date: Thu, 19 May 2022 20:19:25 +0200
Subject: [PATCH 14/17] restore function wrapper approach

---
 src/utilities/request.jl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/utilities/request.jl b/src/utilities/request.jl
index 9615490c83..8bc9914532 100644
--- a/src/utilities/request.jl
+++ b/src/utilities/request.jl
@@ -136,7 +136,7 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
     end
 
     function upgrade_error(f)
-        try
+        return () -> try
             return f()
         catch e
             if e isa HTTP.StatusError
@@ -188,7 +188,7 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
 
     delays = AWSExponentialBackoff(; max_attempts=3)
 
-    retry(upgrade_error ∘ get_response; check=check, delays=delays)()
+    retry(upgrade_error(get_response); check=check, delays=delays)()
 
     if request.use_response_type
         return aws_response

From 6a96faeec98bea24eaf5926c2d0729e6345d1e4a Mon Sep 17 00:00:00 2001
From: Eric Hanson <5846501+ericphanson@users.noreply.github.com>
Date: Thu, 19 May 2022 20:19:41 +0200
Subject: [PATCH 15/17] move `upgrade_error` out of `submit_request`

---
 src/utilities/request.jl | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/utilities/request.jl b/src/utilities/request.jl
index 8bc9914532..85a1e4386f 100644
--- a/src/utilities/request.jl
+++ b/src/utilities/request.jl
@@ -79,6 +79,18 @@ Base.@kwdef mutable struct Request
     response_dict_type::Union{Type{<:AbstractDict},Nothing} = nothing
 end
 
+function upgrade_error(f)
+    return () -> try
+        return f()
+    catch e
+        if e isa HTTP.StatusError
+            e = AWSException(e, stream)
+            rethrow(e)
+        end
+        rethrow()
+    end
+end
+
 """
     submit_request(aws::AbstractAWSConfig, request::Request; return_headers::Bool=false)
 
@@ -135,18 +147,6 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
         end
     end
 
-    function upgrade_error(f)
-        return () -> try
-            return f()
-        catch e
-            if e isa HTTP.StatusError
-                e = AWSException(e, stream)
-                rethrow(e)
-            end
-            rethrow()
-        end
-    end
-
     check = function (s, e)
         # Pass on non-AWS exceptions.
         if !(e isa AWSException)

From f59aa6c582e2e835b27e72dd8242e571565fd8b5 Mon Sep 17 00:00:00 2001
From: Eric Hanson <5846501+ericphanson@users.noreply.github.com>
Date: Thu, 19 May 2022 20:32:20 +0200
Subject: [PATCH 16/17] Revert "move `upgrade_error` out of `submit_request`"

This reverts commit 6a96faeec98bea24eaf5926c2d0729e6345d1e4a.
---
 src/utilities/request.jl | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/utilities/request.jl b/src/utilities/request.jl
index 85a1e4386f..8bc9914532 100644
--- a/src/utilities/request.jl
+++ b/src/utilities/request.jl
@@ -79,18 +79,6 @@ Base.@kwdef mutable struct Request
     response_dict_type::Union{Type{<:AbstractDict},Nothing} = nothing
 end
 
-function upgrade_error(f)
-    return () -> try
-        return f()
-    catch e
-        if e isa HTTP.StatusError
-            e = AWSException(e, stream)
-            rethrow(e)
-        end
-        rethrow()
-    end
-end
-
 """
     submit_request(aws::AbstractAWSConfig, request::Request; return_headers::Bool=false)
 
@@ -147,6 +135,18 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
         end
     end
 
+    function upgrade_error(f)
+        return () -> try
+            return f()
+        catch e
+            if e isa HTTP.StatusError
+                e = AWSException(e, stream)
+                rethrow(e)
+            end
+            rethrow()
+        end
+    end
+
     check = function (s, e)
         # Pass on non-AWS exceptions.
         if !(e isa AWSException)

From 15c6f60e0e9831703171601d6e431d5842953554 Mon Sep 17 00:00:00 2001
From: Eric Hanson <5846501+ericphanson@users.noreply.github.com>
Date: Wed, 25 May 2022 17:01:43 +0200
Subject: [PATCH 17/17] Update test/utilities.jl

Co-authored-by: Curtis Vogt <curtis.vogt@gmail.com>
---
 test/utilities.jl | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/test/utilities.jl b/test/utilities.jl
index b14dc342b0..db6aeaf4c7 100644
--- a/test/utilities.jl
+++ b/test/utilities.jl
@@ -80,14 +80,11 @@ end
     end
 end
 
-# Are we iterating the right number of times?
+# Count the elements in an iterator without using `length`
 function count_len(itr)
     c = 0
-    result = iterate(itr)
-    while result !== nothing
+    for _ in itr
         c += 1
-        state = result[2]
-        result = iterate(itr, state)
     end
     return c
 end