From 62087ad1917f8816a640f58ab2dadeb6a5621cc1 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Wed, 6 Apr 2022 12:25:39 +0200 Subject: [PATCH 1/6] Add `retry=3` to `download()` --- utils/general.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/utils/general.py b/utils/general.py index 5316f504871a..673ed08ad82e 100755 --- a/utils/general.py +++ b/utils/general.py @@ -497,20 +497,27 @@ def url2file(url): return file -def download(url, dir='.', unzip=True, delete=True, curl=False, threads=1): +def download(url, dir='.', unzip=True, delete=True, curl=False, threads=1, retry=3): # Multi-threaded file download and unzip function, used in data.yaml for autodownload def download_one(url, dir): # Download 1 file + success = True f = dir / Path(url).name # filename if Path(url).is_file(): # exists in current path Path(url).rename(f) # move to dir elif not f.exists(): LOGGER.info(f'Downloading {url} to {f}...') - if curl: - os.system(f"curl -L '{url}' -o '{f}' --retry 9 -C -") # curl download, retry and resume on fail - else: - torch.hub.download_url_to_file(url, f, progress=threads == 1) # torch download - if unzip and f.suffix in ('.zip', '.gz'): + for _ in range(retry): + if curl: + r = os.system(f"curl -L '{url}' -o '{f}' --retry 9 -C -") # curl download, retry and resume on fail + success = r == 0 + else: + torch.hub.download_url_to_file(url, f, progress=threads == 1) # torch download + success = f.is_file() + if success: + break + + if unzip and success and f.suffix in ('.zip', '.gz'): LOGGER.info(f'Unzipping {f}...') if f.suffix == '.zip': ZipFile(f).extractall(path=dir) # unzip From 924b0362c426588ff9867181ff9c80edf1bf39d1 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Wed, 6 Apr 2022 12:34:23 +0200 Subject: [PATCH 2/6] Update general.py --- utils/general.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/utils/general.py b/utils/general.py index 673ed08ad82e..66a488816a05 100755 --- a/utils/general.py +++ b/utils/general.py @@ -509,13 +509,16 @@ def download_one(url, dir): LOGGER.info(f'Downloading {url} to {f}...') for _ in range(retry): if curl: - r = os.system(f"curl -L '{url}' -o '{f}' --retry 9 -C -") # curl download, retry and resume on fail + s = 'sS' if threads > 1 else '' # silent + r = os.system(f"curl -{s}L '{url}' -o '{f}' --retry 9 -C -") # curl download success = r == 0 else: torch.hub.download_url_to_file(url, f, progress=threads == 1) # torch download success = f.is_file() if success: break + else: + LOGGER.info(f'Failure, retrying {url} to {f}...') if unzip and success and f.suffix in ('.zip', '.gz'): LOGGER.info(f'Unzipping {f}...') From 0dfb85725b023ec47d1d22e604557a97a40d3e5d Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Wed, 6 Apr 2022 12:38:15 +0200 Subject: [PATCH 3/6] Update general.py --- utils/general.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/utils/general.py b/utils/general.py index 66a488816a05..901e9318dbfb 100755 --- a/utils/general.py +++ b/utils/general.py @@ -507,7 +507,7 @@ def download_one(url, dir): Path(url).rename(f) # move to dir elif not f.exists(): LOGGER.info(f'Downloading {url} to {f}...') - for _ in range(retry): + for i in range(retry): if curl: s = 'sS' if threads > 1 else '' # silent r = os.system(f"curl -{s}L '{url}' -o '{f}' --retry 9 -C -") # curl download @@ -517,8 +517,10 @@ def download_one(url, dir): success = f.is_file() if success: break + elif i < (retry - 1): + LOGGER.info(f'Failure, retrying {i + 1}/{retry} {url}...') else: - LOGGER.info(f'Failure, retrying {url} to {f}...') + LOGGER.info(f'Failed to download {url}...') if unzip and success and f.suffix in ('.zip', '.gz'): LOGGER.info(f'Unzipping {f}...') From d88f4bf2b3b4f2b045a6c38c29c959da8556c3d8 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Wed, 6 Apr 2022 12:40:49 +0200 Subject: [PATCH 4/6] Update general.py --- utils/general.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/general.py b/utils/general.py index 901e9318dbfb..6c2558db74c4 100755 --- a/utils/general.py +++ b/utils/general.py @@ -507,7 +507,7 @@ def download_one(url, dir): Path(url).rename(f) # move to dir elif not f.exists(): LOGGER.info(f'Downloading {url} to {f}...') - for i in range(retry): + for i in range(retry + 1): if curl: s = 'sS' if threads > 1 else '' # silent r = os.system(f"curl -{s}L '{url}' -o '{f}' --retry 9 -C -") # curl download @@ -517,10 +517,10 @@ def download_one(url, dir): success = f.is_file() if success: break - elif i < (retry - 1): - LOGGER.info(f'Failure, retrying {i + 1}/{retry} {url}...') + elif i < retry: + LOGGER.warning(f'Download failure, retrying {i + 1}/{retry} {url}...') else: - LOGGER.info(f'Failed to download {url}...') + LOGGER.warning(f'Failed to download {url}...') if unzip and success and f.suffix in ('.zip', '.gz'): LOGGER.info(f'Unzipping {f}...') From d26f9eec2398b76774df86210c758bac57ea0348 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Wed, 6 Apr 2022 12:47:13 +0200 Subject: [PATCH 5/6] Update VOC.yaml --- data/VOC.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/VOC.yaml b/data/VOC.yaml index fbe3b193bf2e..93a1f181ce8c 100644 --- a/data/VOC.yaml +++ b/data/VOC.yaml @@ -62,7 +62,7 @@ download: | urls = [url + 'VOCtrainval_06-Nov-2007.zip', # 446MB, 5012 images url + 'VOCtest_06-Nov-2007.zip', # 438MB, 4953 images url + 'VOCtrainval_11-May-2012.zip'] # 1.95GB, 17126 images - download(urls, dir=dir / 'images', delete=False, threads=3) + download(urls, dir=dir / 'images', delete=False, curl=True, threads=3) # Convert path = dir / f'images/VOCdevkit' From d385cae4a01d0e83cdc5f47a6260eb6e0e186f69 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Wed, 6 Apr 2022 12:50:44 +0200 Subject: [PATCH 6/6] Update VisDrone.yaml --- data/VisDrone.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/VisDrone.yaml b/data/VisDrone.yaml index ef7e6c4fed35..c38fb2ab769e 100644 --- a/data/VisDrone.yaml +++ b/data/VisDrone.yaml @@ -54,7 +54,7 @@ download: | 'https://github.com/ultralytics/yolov5/releases/download/v1.0/VisDrone2019-DET-val.zip', 'https://github.com/ultralytics/yolov5/releases/download/v1.0/VisDrone2019-DET-test-dev.zip', 'https://github.com/ultralytics/yolov5/releases/download/v1.0/VisDrone2019-DET-test-challenge.zip'] - download(urls, dir=dir, threads=4) + download(urls, dir=dir, curl=True, threads=4) # Convert for d in 'VisDrone2019-DET-train', 'VisDrone2019-DET-val', 'VisDrone2019-DET-test-dev':