Skip to content

Commit

Permalink
Fix restarting downloads (#784)
Browse files Browse the repository at this point in the history
  • Loading branch information
gregtatum authored Aug 1, 2024
1 parent 8ba3952 commit f0c00e2
Showing 1 changed file with 6 additions and 1 deletion.
7 changes: 6 additions & 1 deletion pipeline/common/downloads.py
Original file line number Diff line number Diff line change
Expand Up @@ -299,10 +299,15 @@ def download_chunks(self) -> Generator[bytes, None, None]:
# the retries are done.
logger.error(f"A download error occurred: {error}")

self.close()
# Close out the response on an error. It will be recreated when retrying.
if self.response:
self.response.close()
self.response = None

logger.info(f"Retrying in {self.wait_before_retry_sec} sec")
time.sleep(self.wait_before_retry_sec)

self.close()
raise Exception("The download failed.")

def decode(self, byte_stream) -> Generator[bytes, None, None]:
Expand Down

0 comments on commit f0c00e2

Please sign in to comment.