fix: retry certain http errors reported by replica during asset uploa…

…ds (#3822)

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 # UNRELEASED
 
+### asset uploads: retry some HTTP errors returned by the replica
+
+Now retries the following, with exponential backoff as is already done for connect and transport errors:
+- 500 internal server error
+- 502 bad gateway
+- 503 service unavailable
+- 504 gateway timeout
+- 429 many requests
+
 ### fix: Allow canisters to be deployed even if unrelated canisters in dfx.json are malformed
 
 ### feat!: enable cycles ledger support unconditionally

src/canisters/frontend/ic-asset/src/batch_upload/retryable.rs

@@ -1,8 +1,17 @@
+use ic_agent::agent::http_transport::reqwest_transport::reqwest::StatusCode;
 use ic_agent::AgentError;
 
 pub(crate) fn retryable(agent_error: &AgentError) -> bool {
-    matches!(
-        agent_error,
-        AgentError::TimeoutWaitingForResponse() | AgentError::TransportError(_)
-    )
+    match agent_error {
+        AgentError::TimeoutWaitingForResponse() => true,
+        AgentError::TransportError(_) => true,
+        AgentError::HttpError(http_error) => {
+            http_error.status == StatusCode::INTERNAL_SERVER_ERROR
+                || http_error.status == StatusCode::BAD_GATEWAY
+                || http_error.status == StatusCode::SERVICE_UNAVAILABLE
+                || http_error.status == StatusCode::GATEWAY_TIMEOUT
+                || http_error.status == StatusCode::TOO_MANY_REQUESTS
+        }
+        _ => false,
+    }
 }