[Storage] Add support for FSx Lustre as a shared storage type in us-iso-east-1.

CHANGELOG.md

@@ -6,6 +6,7 @@ CHANGELOG
 
 **ENHANCEMENTS**
 - Add new configuration section `Scheduling/SlurmSettings/ExternalSlurmdbd` to connect the cluster to an external Slurmdbd.
+- Add support for FSx Lustre as a shared storage type in us-iso-east-1.
 
 **BUG FIXES**
 - Fix DRA configuration to make `AutoExportPolicy` and `AutoImportPolicy` optional.

cli/src/pcluster/constants.py

@@ -271,7 +271,7 @@ class Feature(Enum):
 UNSUPPORTED_FEATURES_MAP = {
     Feature.BATCH: ["ap-northeast-3", "us-iso"],
     Feature.DCV: ["us-iso"],
-    Feature.FSX_LUSTRE: ["us-iso"],
+    Feature.FSX_LUSTRE: ["us-isob"],
     Feature.FILE_CACHE: ["us-iso"],
     Feature.FSX_ONTAP: ["us-iso"],
     Feature.FSX_OPENZFS: ["us-iso"],

cli/tests/pcluster/test_utils.py

@@ -574,10 +574,8 @@ async def async_method(self, param):
         (Feature.DCV, "us-iso-west-1", False),
         (Feature.DCV, "us-isob-east-1", False),
         (Feature.DCV, "us-isoWHATEVER", False),
-        (Feature.FSX_LUSTRE, "us-iso-east-1", False),
-        (Feature.FSX_LUSTRE, "us-iso-west-1", False),
         (Feature.FSX_LUSTRE, "us-isob-east-1", False),
-        (Feature.FSX_LUSTRE, "us-isoWHATEVER", False),
+        (Feature.FSX_LUSTRE, "us-isobWHATEVER", False),
         (Feature.FSX_ONTAP, "us-iso-east-1", False),
         (Feature.FSX_ONTAP, "us-iso-west-1", False),
         (Feature.FSX_ONTAP, "us-isob-east-1", False),

tests/integration-tests/tests/ad_integration/test_ad_integration.py

@@ -27,7 +27,15 @@
 from remote_command_executor import RemoteCommandExecutor
 from retrying import retry
 from time_utils import seconds
-from utils import find_stack_by_tag, generate_stack_name, is_directory_supported, is_fsx_supported, random_alphanumeric
+from utils import (
+    find_stack_by_tag,
+    generate_stack_name,
+    is_directory_supported,
+    is_fsx_lustre_supported,
+    is_fsx_ontap_supported,
+    is_fsx_openzfs_supported,
+    random_alphanumeric,
+)
 
 from tests.ad_integration.cluster_user import ClusterUser
 from tests.common.utils import run_system_analyzer
@@ -513,7 +521,7 @@ def _check_ssh_key(user, ssh_generation_enabled, remote_command_executor, schedu
     ],
 )
 @pytest.mark.usefixtures("os", "instance")
-def test_ad_integration(
+def test_ad_integration(  # noqa: C901
     region,
     scheduler,
     scheduler_commands_factory,
@@ -541,9 +549,14 @@ def test_ad_integration(
     if not is_directory_supported(region, directory_type):
         pytest.skip(f"Skipping the test because directory type {directory_type} is not supported in region {region}")
 
-    fsx_supported = is_fsx_supported(region)
+    fsx_lustre_supported = is_fsx_lustre_supported(region)
+    fsx_ontap_supported = is_fsx_ontap_supported(region)
+    fsx_openzfs_supported = is_fsx_openzfs_supported(region)
+
     config_params = {
-        "fsx_supported": fsx_supported,
+        "fsx_lustre_supported": fsx_lustre_supported,
+        "fsx_ontap_supported": fsx_ontap_supported,
+        "fsx_openzfs_supported": fsx_openzfs_supported,
     }
     directory_stack_name = directory_factory(
         request.config.getoption("directory_stack_name"),
@@ -616,8 +629,12 @@ def test_ad_integration(
             )
         )
     shared_storage_mount_dirs = ["/shared", "/efs"]
-    if fsx_supported:
-        shared_storage_mount_dirs.extend(["/fsxlustre", "/fsxontap", "/fsxopenzfs"])
+    if fsx_lustre_supported:
+        shared_storage_mount_dirs.extend(["/fsxlustre"])
+    if fsx_ontap_supported:
+        shared_storage_mount_dirs.extend(["/fsxontap"])
+    if fsx_openzfs_supported:
+        shared_storage_mount_dirs.extend(["/fsxopenzfs"])
     _run_user_workloads(users, test_datadir, shared_storage_mount_dirs)
     logging.info("Testing pcluster update and generate ssh keys for user")
     _check_ssh_key_generation(users[0], remote_command_executor, scheduler_commands, False)

tests/integration-tests/tests/ad_integration/test_ad_integration/test_ad_integration/pcluster.config.update.yaml

@@ -33,17 +33,21 @@ SharedStorage:
   - MountDir: /efs
     Name: efs
     StorageType: Efs
-  {% if fsx_supported %}
+  {% if fsx_lustre_supported %}
   - MountDir: /fsxlustre
     Name: fsx
     StorageType: FsxLustre
     FsxLustreSettings:
       StorageCapacity: 2400
+  {% endif %}
+  {% if fsx_openzfs_supported %}
   - MountDir: /fsxopenzfs
     Name: existingopenzfs
     StorageType: FsxOpenZfs
     FsxOpenZfsSettings:
       VolumeId: {{ fsx_open_zfs_volume_id }}
+  {% endif %}
+  {% if fsx_ontap_supported %}
   - MountDir: /fsxontap
     Name: existingontap
     StorageType: FsxOntap

tests/integration-tests/tests/ad_integration/test_ad_integration/test_ad_integration/pcluster.config.update2.yaml

@@ -33,17 +33,21 @@ SharedStorage:
   - MountDir: /efs
     Name: efs
     StorageType: Efs
-  {% if fsx_supported %}
+  {% if fsx_lustre_supported %}
   - MountDir: /fsxlustre
     Name: fsx
     StorageType: FsxLustre
     FsxLustreSettings:
       StorageCapacity: 2400
+  {% endif %}
+  {% if fsx_openzfs_supported %}
   - MountDir: /fsxopenzfs
     Name: existingopenzfs
     StorageType: FsxOpenZfs
     FsxOpenZfsSettings:
       VolumeId: {{ fsx_open_zfs_volume_id }}
+  {% endif %}
+  {% if fsx_ontap_supported %}
   - MountDir: /fsxontap
     Name: existingontap
     StorageType: FsxOntap

tests/integration-tests/tests/ad_integration/test_ad_integration/test_ad_integration/pcluster.config.yaml

@@ -33,17 +33,21 @@ SharedStorage:
   - MountDir: /efs
     Name: efs
     StorageType: Efs
-  {% if fsx_supported %}
+  {% if fsx_lustre_supported %}
   - MountDir: /fsxlustre
     Name: fsx
     StorageType: FsxLustre
     FsxLustreSettings:
       StorageCapacity: 2400
+  {% endif %}
+  {% if fsx_openzfs_supported %}
   - MountDir: /fsxopenzfs
     Name: existingopenzfs
     StorageType: FsxOpenZfs
     FsxOpenZfsSettings:
       VolumeId: {{ fsx_open_zfs_volume_id }}
+  {% endif %}
+  {% if fsx_ontap_supported %}
   - MountDir: /fsxontap
     Name: existingontap
     StorageType: FsxOntap

tests/integration-tests/tests/ad_integration/test_ad_integration/test_ad_integration_on_login_nodes/pcluster.config.yaml

@@ -43,17 +43,21 @@ SharedStorage:
   - MountDir: /efs
     Name: efs
     StorageType: Efs
-  {% if fsx_supported %}
+  {% if fsx_lustre_supported %}
   - MountDir: /fsxlustre
     Name: fsx
     StorageType: FsxLustre
     FsxLustreSettings:
       StorageCapacity: 2400
+  {% endif %}
+  {% if fsx_openzfs_supported %}
   - MountDir: /fsxopenzfs
     Name: existingopenzfs
     StorageType: FsxOpenZfs
     FsxOpenZfsSettings:
       VolumeId: {{ fsx_open_zfs_volume_id }}
+  { % endif % }
+  { % if fsx_ontap_supported % }
   - MountDir: /fsxontap
     Name: existingontap
     StorageType: FsxOntap

tests/integration-tests/tests/storage/test_shared_home.py

@@ -15,7 +15,7 @@
 import pytest
 from assertpy import assert_that
 from remote_command_executor import RemoteCommandExecutor
-from utils import is_fsx_supported
+from utils import is_fsx_lustre_supported, is_fsx_ontap_supported, is_fsx_openzfs_supported
 
 from tests.storage.storage_common import (
     check_fsx,
@@ -57,8 +57,13 @@ def test_shared_home(
     """Verify the shared /home storage fs is available when set"""
     mount_dir = "/home"
     bucket_name = None
-    if is_fsx_supported(region) or storage_type in ["Efs", "Ebs"]:
-        if storage_type == "FsxOpenZfs":
+
+    fsx_supported = (
+        is_fsx_lustre_supported(region) or is_fsx_ontap_supported(region) or is_fsx_openzfs_supported(region)
+    )
+
+    if fsx_supported or storage_type in ["Efs", "Ebs"]:
+        if is_fsx_openzfs_supported(region) and storage_type == "FsxOpenZfs":
             fsx_open_zfs_root_volume_id = create_fsx_open_zfs(fsx_factory, num=1)[0]
             fsx_open_zfs_volume_id = open_zfs_volume_factory(fsx_open_zfs_root_volume_id, num_volumes=1)[0]
             cluster_config = pcluster_config_reader(
@@ -67,7 +72,7 @@ def test_shared_home(
                 volume_id=fsx_open_zfs_volume_id,
                 shared_storage_type=shared_storage_type,
             )
-        elif storage_type == "FsxOntap":
+        elif is_fsx_ontap_supported(region) and storage_type == "FsxOntap":
             fsx_ontap_fs_id = create_fsx_ontap(fsx_factory, num=1)[0]
             fsx_on_tap_volume_id = svm_factory(fsx_ontap_fs_id, num_volumes=1)[0]
             cluster_config = pcluster_config_reader(