From ff481cd3dbc47e492117d0dcdc89349492c2d952 Mon Sep 17 00:00:00 2001 From: Hanwen Date: Wed, 5 Jun 2024 10:02:24 -0700 Subject: [PATCH] Decouple region support from CLI release Remove the constant variable and move the content to a file. This file will be uploaded to official S3 buckets during CLI release and/or region expansion. If the file in S3 is not accessible, the code falls back to the local file. The file in S3 is not accessible when we are developing new versions or network access is limited Signed-off-by: Hanwen --- cli/src/pcluster/api/controllers/common.py | 14 +++++---- .../pcluster/cli/commands/configure/utils.py | 6 ++-- cli/src/pcluster/constants.py | 29 ------------------- cli/src/pcluster/resources/supported-regions | 27 +++++++++++++++++ cli/src/pcluster/utils.py | 21 ++++++++++++++ .../pcluster/validators/cluster_validators.py | 4 +-- 6 files changed, 63 insertions(+), 38 deletions(-) create mode 100644 cli/src/pcluster/resources/supported-regions diff --git a/cli/src/pcluster/api/controllers/common.py b/cli/src/pcluster/api/controllers/common.py index c3a914a5c8..209d4e37e0 100644 --- a/cli/src/pcluster/api/controllers/common.py +++ b/cli/src/pcluster/api/controllers/common.py @@ -29,10 +29,10 @@ ) from pcluster.aws.common import BadRequestError, LimitExceededError, StackNotFoundError, get_region from pcluster.config.common import AllValidatorsSuppressor, TypeMatchValidatorsSuppressor, ValidatorSuppressor -from pcluster.constants import SUPPORTED_REGIONS, UNSUPPORTED_OPERATIONS_MAP, Operation +from pcluster.constants import UNSUPPORTED_OPERATIONS_MAP, Operation from pcluster.models.cluster import Cluster from pcluster.models.common import BadRequest, Conflict, LimitExceeded, NotFound, parse_config -from pcluster.utils import get_installed_version, to_utc_datetime +from pcluster.utils import get_installed_version, retrieve_supported_regions, to_utc_datetime LOGGER = logging.getLogger(__name__) @@ -40,11 +40,15 @@ def _set_region(region): if not region: raise BadRequestException("region needs to be set") - if region not in SUPPORTED_REGIONS: - raise BadRequestException(f"invalid or unsupported region '{region}'") - + region_backup = os.environ.get("AWS_DEFAULT_REGION") LOGGER.info("Setting AWS Region to %s", region) os.environ["AWS_DEFAULT_REGION"] = region + if region not in retrieve_supported_regions(): + if region_backup: + os.environ["AWS_DEFAULT_REGION"] = region_backup + else: + del os.environ["AWS_DEFAULT_REGION"] + raise BadRequestException(f"invalid or unsupported region '{region}'") def configure_aws_region_from_config(region: Union[None, str], config_str: str): diff --git a/cli/src/pcluster/cli/commands/configure/utils.py b/cli/src/pcluster/cli/commands/configure/utils.py index 6a4d377b75..613b9b094d 100644 --- a/cli/src/pcluster/cli/commands/configure/utils.py +++ b/cli/src/pcluster/cli/commands/configure/utils.py @@ -19,7 +19,7 @@ from pcluster.aws.aws_api import AWSApi from pcluster.aws.common import AWSClientError -from pcluster.constants import SUPPORTED_REGIONS +from pcluster.utils import retrieve_supported_regions LOGGER = logging.getLogger(__name__) @@ -201,7 +201,9 @@ def get_rows_and_header(items): def get_regions(): ec2 = boto3.client("ec2") regions = ec2.describe_regions().get("Regions") - regions = [region.get("RegionName") for region in regions if region.get("RegionName") in SUPPORTED_REGIONS] + regions = [ + region.get("RegionName") for region in regions if region.get("RegionName") in retrieve_supported_regions() + ] regions.sort() return regions diff --git a/cli/src/pcluster/constants.py b/cli/src/pcluster/constants.py index f774d7a766..9f7e11f816 100644 --- a/cli/src/pcluster/constants.py +++ b/cli/src/pcluster/constants.py @@ -198,35 +198,6 @@ PCLUSTER_S3_BUCKET_VERSION = "v1" -SUPPORTED_REGIONS = [ - "af-south-1", - "ap-east-1", - "ap-northeast-1", - "ap-northeast-2", - "ap-south-1", - "ap-southeast-1", - "ap-southeast-2", - "ca-central-1", - "cn-north-1", - "cn-northwest-1", - "eu-central-1", - "eu-north-1", - "eu-south-1", - "eu-west-1", - "eu-west-2", - "eu-west-3", - "il-central-1", - "me-south-1", - "sa-east-1", - "us-east-1", - "us-east-2", - "us-iso-east-1", - "us-isob-east-1", - "us-gov-east-1", - "us-gov-west-1", - "us-west-1", - "us-west-2", -] # see https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html NODEJS_MIN_VERSION = "10.13.0" diff --git a/cli/src/pcluster/resources/supported-regions b/cli/src/pcluster/resources/supported-regions new file mode 100644 index 0000000000..68ab8abe50 --- /dev/null +++ b/cli/src/pcluster/resources/supported-regions @@ -0,0 +1,27 @@ +af-south-1 +ap-east-1 +ap-northeast-1 +ap-northeast-2 +ap-south-1 +ap-southeast-1 +ap-southeast-2 +ca-central-1 +cn-north-1 +cn-northwest-1 +eu-central-1 +eu-north-1 +eu-south-1 +eu-west-1 +eu-west-2 +eu-west-3 +il-central-1 +me-south-1 +sa-east-1 +us-east-1 +us-east-2 +us-iso-east-1 +us-isob-east-1 +us-gov-east-1 +us-gov-west-1 +us-west-1 +us-west-2 \ No newline at end of file diff --git a/cli/src/pcluster/utils.py b/cli/src/pcluster/utils.py index 8426b4d9dc..a94608b4f2 100644 --- a/cli/src/pcluster/utils.py +++ b/cli/src/pcluster/utils.py @@ -20,13 +20,16 @@ import string import sys import time +import urllib import zipfile from concurrent.futures import ThreadPoolExecutor from io import BytesIO from shlex import quote from typing import Callable, NoReturn +from urllib.error import URLError from urllib.parse import urlparse +import boto3 import dateutil.parser import pkg_resources import yaml @@ -558,3 +561,21 @@ async def wrapper(self, *args, **kwargs): ) return wrapper + + +def retrieve_supported_regions(): + """Retrieve the list of supported regions.""" + if not hasattr(retrieve_supported_regions, "cache"): + region = boto3.Session().region_name or "us-east-1" + try: + url = "https://{region}-aws-parallelcluster.s3.{region}.{aws_domain}/supported-regions".format( + region=region, + aws_domain=get_partition(region), + ) + with urllib.request.urlopen(url) as f: # nosec B310 nosemgrep + retrieve_supported_regions.cache = f.read().decode("utf-8").split("\n") + except URLError: + # When the file is not found on the URL, use local file. This is useful when developing new versions. + with open(pkg_resources.resource_filename(__name__, "/resources/supported-regions"), encoding="utf-8") as f: + retrieve_supported_regions.cache = f.read().split("\n") + return retrieve_supported_regions.cache diff --git a/cli/src/pcluster/validators/cluster_validators.py b/cli/src/pcluster/validators/cluster_validators.py index 15cb1d62ce..1594d69cb2 100644 --- a/cli/src/pcluster/validators/cluster_validators.py +++ b/cli/src/pcluster/validators/cluster_validators.py @@ -36,7 +36,6 @@ RETAIN_POLICY, SCHEDULERS_SUPPORTING_IMDS_SECURED, SUPPORTED_OSES, - SUPPORTED_REGIONS, SUPPORTED_SCHEDULERS, ) from pcluster.launch_template_utils import _LaunchTemplateBuilder @@ -45,6 +44,7 @@ get_supported_os_for_architecture, get_supported_os_for_scheduler, remove_none_values, + retrieve_supported_regions, ) from pcluster.validators.common import FailureLevel, Validator @@ -117,7 +117,7 @@ class RegionValidator(Validator): """Region validator.""" def _validate(self, region): - if region not in SUPPORTED_REGIONS: + if region not in retrieve_supported_regions(): self._add_failure( f"Region '{region}' is not yet officially supported by ParallelCluster", FailureLevel.ERROR )