From 15791a7df099975b8f4667be195825f4c1b476b1 Mon Sep 17 00:00:00 2001 From: Martin Klefas-Stennett Date: Thu, 14 Sep 2023 21:33:08 +0100 Subject: [PATCH 1/4] explicit region_name in boto3 client init --- runpod/serverless/utils/rp_upload.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/runpod/serverless/utils/rp_upload.py b/runpod/serverless/utils/rp_upload.py index af6ce4cc..c3958611 100644 --- a/runpod/serverless/utils/rp_upload.py +++ b/runpod/serverless/utils/rp_upload.py @@ -23,6 +23,17 @@ FMT = "%(filename)-20s:%(lineno)-4d %(asctime)s %(message)s" logging.basicConfig(level=logging.INFO, format=FMT, handlers=[logging.StreamHandler()]) +def extract_region_from_url(endpoint_url): + # AWS/backblaze S3-like URL + if '.s3.' in endpoint_url: + return endpoint_url.split('.s3.')[1].split('.')[0] + # DigitalOcean Spaces-like URL + elif '.digitaloceanspaces.com' in endpoint_url: + return endpoint_url.split('.')[1].split('.digitaloceanspaces.com')[0] + else: + # Additional cases can be added here + return None + # --------------------------- S3 Bucket Connection --------------------------- # def get_boto_client( @@ -57,12 +68,16 @@ def get_boto_client( secret_access_key = os.environ.get('BUCKET_SECRET_ACCESS_KEY', None) if endpoint_url and access_key_id and secret_access_key: + # Extract region from the endpoint URL + region = extract_region_from_url(endpoint_url) + boto_client = bucket_session.client( 's3', endpoint_url=endpoint_url, aws_access_key_id=access_key_id, aws_secret_access_key=secret_access_key, - config=boto_config + config=boto_config, + region_name=region ) else: boto_client = None From 2a463114b68f798c126c81a6513e54d9d36aaa24 Mon Sep 17 00:00:00 2001 From: Martin Klefas-Stennett Date: Fri, 15 Sep 2023 21:22:28 +0100 Subject: [PATCH 2/4] properly sanitize url substring --- runpod/serverless/utils/rp_upload.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/runpod/serverless/utils/rp_upload.py b/runpod/serverless/utils/rp_upload.py index c3958611..f56484ae 100644 --- a/runpod/serverless/utils/rp_upload.py +++ b/runpod/serverless/utils/rp_upload.py @@ -18,17 +18,19 @@ from boto3.s3.transfer import TransferConfig from botocore.config import Config from tqdm_loggable.auto import tqdm +from urllib.parse import urlparse logger = logging.getLogger("runpod upload utility") FMT = "%(filename)-20s:%(lineno)-4d %(asctime)s %(message)s" logging.basicConfig(level=logging.INFO, format=FMT, handlers=[logging.StreamHandler()]) def extract_region_from_url(endpoint_url): + parsed_url = urlparse(endpoint_url) # AWS/backblaze S3-like URL if '.s3.' in endpoint_url: return endpoint_url.split('.s3.')[1].split('.')[0] # DigitalOcean Spaces-like URL - elif '.digitaloceanspaces.com' in endpoint_url: + elif parsed_url.netloc.endswith('.digitaloceanspaces.com'): return endpoint_url.split('.')[1].split('.digitaloceanspaces.com')[0] else: # Additional cases can be added here From 19f98dbf31a652720893e90cba89c3d9816dc3dd Mon Sep 17 00:00:00 2001 From: Martin Klefas-Stennett Date: Fri, 15 Sep 2023 23:53:03 +0100 Subject: [PATCH 3/4] update upload test case to include s3 region_name --- tests/test_serverless/test_utils/test_upload.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_serverless/test_utils/test_upload.py b/tests/test_serverless/test_utils/test_upload.py index e36f67e6..ef3b0e28 100644 --- a/tests/test_serverless/test_utils/test_upload.py +++ b/tests/test_serverless/test_utils/test_upload.py @@ -58,7 +58,8 @@ def test_get_boto_client(self): endpoint_url=bucket_creds['endpointUrl'], aws_access_key_id=bucket_creds['accessId'], aws_secret_access_key=bucket_creds['accessSecret'], - config=unittest.mock.ANY + config=unittest.mock.ANY, + region_name=None ) def test_get_boto_client_environ(self): From 36f20ffef3f249a18dcfacd1df5319463d9684d4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 10:56:17 +0000 Subject: [PATCH 4/4] Bump nest-asyncio from 1.5.7 to 1.5.8 Bumps [nest-asyncio](https://github.com/erdewit/nest_asyncio) from 1.5.7 to 1.5.8. - [Commits](https://github.com/erdewit/nest_asyncio/compare/v1.5.7...v1.5.8) --- updated-dependencies: - dependency-name: nest-asyncio dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 002352d6..1de74620 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,4 +18,4 @@ fastapi[all] == 0.103.1 urllib3 >= 1.26.6 # Testing Requirements -nest_asyncio == 1.5.7 +nest_asyncio == 1.5.8