feat(aws-pcs): add integrated monitoring, P6 GPU node groups, and container runtime to the PCS reference cluster

architectures/aws-pcs/assets/add-cng-p5.yaml

@@ -1,5 +1,23 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: MIT-0
+#
+# Why a separate add-cng template per GPU family (p5 / p6-b200 / p6-b300)?
+# The templates are ~85% identical; the real difference is the NetworkInterfaces
+# block, whose EFA topology differs per family and cannot be parameterized in
+# plain CloudFormation:
+#   - p5 / p5e / p5en: card 0 = EFA, EFA cards use DeviceIndex 1, 32 or 16 cards
+#     (this template already serves all three via the Use32Interfaces condition).
+#   - p6-b200:         card 0 = EFA, DeviceIndex 1, 8 cards.
+#   - p6-b300:         card 0 = ENA-only (no EFA), EFA on cards 1-16 with
+#                      DeviceIndex 0, 17 cards.
+# A single template would need ~32 per-card !If wrappers plus card-0/DeviceIndex
+# branching, which obscures the list and means editing the already-validated
+# p5/p6-b200 blocks. Fn::ForEach (AWS::LanguageExtensions) could generate the list
+# from a mapping, but that transform requires CAPABILITY_AUTO_EXPAND, which breaks
+# the README/workshop one-click quick-create links. So each family keeps its own
+# flat, hand-checkable NIC list. Consolidating via Fn::ForEach is tracked in
+# docs/ROADMAP.md. InstanceType is locked with AllowedValues so a type whose NIC
+# layout doesn't match this template's block can't be selected.
 
 AWSTemplateFormatVersion: '2010-09-09'
 Description: Adding Compute Node Group to PCS Cluster with multi-network interface support for P5 series (On-Demand or Capacity Block)
@@ -17,7 +35,6 @@ Metadata:
           - CngName
           - QueueName
           - InstanceType
-          - NetworkInterfaceCount
           - MaxCount
           - MinCount
       - Label:
@@ -31,16 +48,37 @@ Metadata:
           - FSxLustreFilesystemId
           - FSxLustreFilesystemMountName
           - FSxOpenZFSFilesystemId
+      - Label:
+          default: Monitoring configuration
+        Parameters:
+          - DeployMonitoring
+          - MonitoringVersion
+          - MonitoringRepo
+          - MonitoringRole
+      - Label:
+          default: Post-install script (OnNodeConfigured)
+        Parameters:
+          - PostInstallScriptUrl
+          - PostInstallScriptArgs
 
 Parameters:
   CapacityReservationId:
     Type: String
-    Description: (Optional) Capacity Reservation ID for Capacity Blocks. Leave empty for On-Demand instances.
+    Description: >-
+      (Optional) Capacity Block for ML reservation ID. When SET, the node group
+      launches with MarketType=capacity-block against this reservation. Leave
+      EMPTY for On-Demand. NOTE: this is for Capacity Blocks ONLY — do not put an
+      On-Demand Capacity Reservation (ODCR) ID here. An ODCR with "open" matching
+      is consumed automatically by On-Demand launches, so for ODCR leave this empty.
     Default: ""
 
   ClusterId:
     Type: String
-    Description: Cluster ID
+    Description: Cluster ID (actual PCS cluster ID like pcs_xxxxx)
+
+  ClusterName:
+    Type: String
+    Description: User-friendly cluster name
 
   CngName:
     Type: String
@@ -54,16 +92,23 @@ Parameters:
 
   InstanceType:
     Type: String
-    Description: Instance Type for Compute Node Group (e.g., p5.48xlarge, p5e.48xlarge, p5en.48xlarge)
+    Description: >-
+      P5-family instance type. The EFA interface count is derived automatically:
+      p5en.48xlarge = 16 network cards, p5.48xlarge / p5e.48xlarge = 32.
     Default: "p5.48xlarge"
-
-  NetworkInterfaceCount:
-    Type: String
-    Description: Number of network interfaces for P5 instances (16 for p5en.48xlarge, 32 for p5.48xlarge and p5e.48xlarge)
-    Default: "32"
     AllowedValues:
-      - "16"
-      - "32"
+      - p5.48xlarge
+      - p5e.48xlarge
+      - p5en.48xlarge
+
+  RootVolumeSize:
+    Type: Number
+    Description: >-
+      Root EBS volume size (GiB) for compute nodes. The PCS-Ready DLAMI default (~75
+      GiB) is too small for pulling/importing large container images (Enroot/Pyxis
+      squashfs, Megatron ~20 GB). 300 GiB gives ample headroom.
+    Default: 300
+    MinValue: 75
 
   MaxCount:
     Type: Number
@@ -78,9 +123,14 @@ Parameters:
     Description: Select the Subnet where the Compute Node will be launched
 
   AmiId:
-    Type: AWS::EC2::Image::Id
-    Description: Enter the AMI ID for the Login Node and Compute Node (e.g. ami-0abcdef1234567890)
-    ConstraintDescription: Must be a valid AMI ID (ami-xxxxxxxxxxxxxxxxx)
+    Type: String
+    Description: >-
+      (Optional) AMI ID for the nodes. Leave empty to auto-resolve the latest
+      PCS-Ready DLAMI (Ubuntu 24.04 x86_64) from SSM Parameter Store. Set an explicit
+      ami-xxxx only to override (e.g. a custom AMI built with Enroot/Pyxis pre-baked).
+    Default: ""
+    AllowedPattern: '^$|^ami-[0-9a-f]{8,17}$'
+    ConstraintDescription: Must be empty or a valid AMI ID (ami-xxxxxxxxxxxxxxxxx)
 
   ClusterSecurityGroupId:
     Type: AWS::EC2::SecurityGroup::Id
@@ -102,14 +152,91 @@ Parameters:
     Type: String
     Description: Filesystem ID for FSx for OpenZFS
 
+  DeployMonitoring:
+    Type: String
+    Description: Deploy monitoring stack (Prometheus/Grafana on Login Node, DCGM exporter on Compute Node with GPU)
+    Default: 'false'
+    AllowedValues:
+      - 'true'
+      - 'false'
+
+  MonitoringVersion:
+    Type: String
+    Description: >-
+      aws-parallelcluster-monitoring git ref to install (release tag, branch, or
+      'latest'). Used both to fetch post-install.sh from MonitoringRepo at this
+      ref and as the ref argument passed to it. Pin to a release tag for upstream
+      stability in production; a branch name (e.g. a fork's dev branch) can be
+      used together with MonitoringRepo for testing unreleased changes.
+    Default: 'v2.9.1'
+
+  MonitoringRepo:
+    Type: String
+    Description: >-
+      GitHub "owner/repo" to fetch the monitoring stack from. Override with a fork
+      together with a branch in MonitoringVersion to test unreleased changes before
+      they merge upstream.
+    Default: 'aws-samples/aws-parallelcluster-monitoring'
+
+  DcgmExporterImage:
+    Type: String
+    Description: >-
+      dcgm-exporter container image used by the monitoring stack on GPU nodes. The
+      default is a DCGM 4.5.2 build pinned by DIGEST -- this covers Hopper / B200 /
+      B300 alike (the digest pull bypasses the Docker 29.x OCI-index failure that
+      breaks newer NVCR tags). To pin to the older monitoring-default 4.2.0 (covers
+      Hopper + B200 only) or any other build, set this to that image reference
+      (preferably also a digest). No effect on CPU nodes.
+    Default: 'nvcr.io/nvidia/k8s/dcgm-exporter@sha256:a7ad6547d4546eaf4dd5d6b4c0b4db4101e63ef7dc3cdff7f42b767d2c60b706'
+
+  MonitoringRole:
+    Type: String
+    Description: Monitoring role for this node (login=head node with Prometheus/Grafana, compute=compute node with exporters, none=no monitoring)
+    Default: 'none'
+    AllowedValues:
+      - 'login'
+      - 'compute'
+      - 'none'
+
+  PostInstallScriptUrl:
+    Type: String
+    Description: >-
+      Optional URL of a post-install script to download and run on each node at
+      first boot (PCS equivalent of ParallelCluster OnNodeConfigured). Leave
+      empty to skip. To install Enroot/Pyxis, point this at scripts/install-enroot-pyxis.sh.
+      Must be an HTTP(S) URL (e.g. a GitHub raw URL); S3 public hosting is not allowed.
+    Default: ''
+
+  PostInstallScriptArgs:
+    Type: String
+    Description: Space-separated arguments passed to the post-install script.
+    Default: ''
+
+  SlurmVersion:
+    Type: String
+    Description: >-
+      The cluster's Slurm version, exported to the post-install script as
+      PCS_SLURM_VERSION so it can put the matching Slurm CLI bin on slurmd's PATH
+      (the Pyxis PMI hook needs scontrol). Must match the version the cluster was
+      created with.
+    Default: '25.11'
+    AllowedValues:
+      - '25.05'
+      - '25.11'
+
 Conditions:
-  Use32Interfaces:
-    Fn::Equals:
-      - Ref: NetworkInterfaceCount
-      - "32"
+  # EFA interface count is derived from the instance type: p5en.48xlarge has 16
+  # network cards, p5.48xlarge / p5e.48xlarge have 32. So "use 32 interfaces"
+  # means "not p5en". No separate NetworkInterfaceCount parameter is needed.
+  Use32Interfaces: !Not [!Equals [!Ref InstanceType, "p5en.48xlarge"]]
   UseCapacityBlock: !Not [!Equals [!Ref CapacityReservationId, ""]]
   UseOnDemand: !Equals [!Ref CapacityReservationId, ""]
   CreateQueue: !Not [!Equals [!Ref QueueName, ""]]
+  # Emit the monitoring-role tag for both login and compute nodes (any role
+  # other than 'none'). The monitoring stack identifies login nodes by this
+  # tag (monitoring-role=login) and excludes them from EC2 service discovery.
+  IsMonitoringRoleSet: !Not [!Equals [!Ref MonitoringRole, 'none']]
+  HasAmiId: !Not [!Equals [!Ref AmiId, ""]]
 
 Resources:
 
@@ -132,10 +259,16 @@ Resources:
       IamInstanceProfileArn: !Ref IamProfileArn
       CustomLaunchTemplate:
         TemplateId: !Ref PCSLaunchTemplate
-        Version: 1
+        # Track the latest launch template version so template updates (e.g. NIC
+        # or UserData changes) actually reach the node group. Hardcoding Version: 1
+        # pins the node group to the original version and silently ignores updates.
+        Version: !GetAtt PCSLaunchTemplate.LatestVersionNumber
       SubnetIds:
         - !Ref SubnetId
-      AmiId: !Ref AmiId
+      AmiId: !If
+        - HasAmiId
+        - !Ref AmiId
+        - '{{resolve:ssm:/aws/service/pcs/ami/dlami-base-ubuntu2404/x86_64/latest/ami-id}}'
       InstanceConfigs:
         - InstanceType: !Ref InstanceType
       PurchaseOption: !If [UseCapacityBlock, CAPACITY_BLOCK, !Ref "AWS::NoValue"]
@@ -156,6 +289,12 @@ Resources:
 
       LaunchTemplateData:
         InstanceType: !Ref InstanceType
+        BlockDeviceMappings:
+          - DeviceName: /dev/sda1
+            Ebs:
+              VolumeSize: !Ref RootVolumeSize
+              VolumeType: gp3
+              DeleteOnTermination: true
         InstanceMarketOptions: !If
           - UseCapacityBlock
           - MarketType: capacity-block
@@ -174,12 +313,34 @@ Resources:
             Tags:
               - Key: HPCRecipes
                 Value: "true"
+              # Name is free for arbitrary operator use. The monitoring stack
+              # identifies login vs compute nodes via the monitoring-role tag
+              # (below), not Name, so dashboards work regardless of this value.
+              # Default to PCS-<cngname> (mirrors CngName) for a recognizable
+              # console name; retag freely afterward.
               - Key: Name
                 Value: !Sub 'PCS-${CngName}'
+              - Key: CngName
+                Value: !Sub 'PCS-${CngName}'
+              - Key: ClusterName
+                Value: !Ref ClusterName
+              - Key: pcs-cluster-id
+                Value: !Ref ClusterId
+              # Node type for the monitoring stack. Emitted for both login and
+              # compute (any role != none). Prometheus drops monitoring-role=login
+              # from EC2 SD (the static login_node job scrapes it); everything else
+              # is reported as instance_name=Compute. See aws-parallelcluster-monitoring
+              # prometheus-pcs.yml and issue #45.
+              - !If
+                - IsMonitoringRoleSet
+                - Key: monitoring-role
+                  Value: !Ref MonitoringRole
+                - !Ref AWS::NoValue
         MetadataOptions:
           HttpEndpoint: enabled
           HttpPutResponseHopLimit: 4
           HttpTokens: required
+          InstanceMetadataTags: enabled
         UserData:
           Fn::Base64: !Sub |
             MIME-Version: 1.0
@@ -190,15 +351,51 @@ Resources:
             MIME-Version: 1.0
 
             runcmd:
+            # Post-install script (optional) - generic OnNodeConfigured-style hook.
+            # Downloads and runs a user-supplied script (e.g. Enroot/Pyxis install).
+            - |
+              if [ -n "${PostInstallScriptUrl}" ]; then
+                echo "Downloading post-install script from ${PostInstallScriptUrl}..." | tee /var/log/pcs-post-install.log
+                curl -fsSL "${PostInstallScriptUrl}" -o /tmp/pcs-post-install.sh
+                chmod +x /tmp/pcs-post-install.sh
+                echo "Executing post-install script..." | tee -a /var/log/pcs-post-install.log
+                # Tell the script this cluster's Slurm version (it can't discover it at
+                # first boot, before slurmd/profile.d/controller exist). See script header.
+                export PCS_SLURM_VERSION="${SlurmVersion}"
+                bash /tmp/pcs-post-install.sh ${PostInstallScriptArgs} >> /var/log/pcs-post-install.log 2>&1
+                echo "Post-install script completed (exit $?)" | tee -a /var/log/pcs-post-install.log
+              else
+                echo "No post-install script configured (PostInstallScriptUrl empty)" | tee /var/log/pcs-post-install.log
+              fi
+            # FSx Home directories mount
             - mkdir -p /tmp/home
             - rsync -aA /home/ /tmp/home
             - echo "${FSxOpenZFSFilesystemId}.fsx.${AWS::Region}.amazonaws.com:/fsx/ /home nfs noatime,nfsvers=3,sync,nconnect=16,rsize=1048576,wsize=1048576,defaults 0 0" >> /etc/fstab
             - mount -a -t nfs defaults
-            - if [ "enabled" == "$(sestatus | awk '/^SELinux status:/{print $3}')" ]; then setsebool -P use_nfs_home_dirs 1; fi
+            - if [ "enabled" = "$(sestatus | awk '/^SELinux status:/{print $3}')" ]; then setsebool -P use_nfs_home_dirs 1; fi
             - rsync -aA --ignore-existing /tmp/home/ /home
             - rm -rf /tmp/home/
             # If provided, mount FSxL filesystem as /fsx
             - if [ ! -z "${FSxLustreFilesystemId}" ]; then mkdir -p /fsx; mount -t lustre ${FSxLustreFilesystemId}.fsx.${AWS::Region}.amazonaws.com@tcp:/${FSxLustreFilesystemMountName} /fsx; chmod 1777 /fsx; fi
+            # Monitoring stack installation (optional, enabled on Login Node only)
+            - |
+              if [ "${DeployMonitoring}" = "true" ]; then
+                echo "Installing monitoring stack..."
+                # Fetch post-install.sh from MonitoringRepo at the MonitoringVersion ref, and
+                # pass both the ref and the repo to it so the tarball is pulled from the same
+                # source. This lets a fork + branch be used for testing unreleased changes.
+                # As of v2.6.3 the installer auto-detects the Ubuntu 'ubuntu' user and runs
+                # install.sh itself (PR #44), so no ec2-user shim or local-var patch is needed.
+                # Optional dcgm-exporter image override (e.g. a B300-capable DCGM
+                # >= 4.4.0 build by digest). The monitoring installer reads
+                # DCGM_EXPORTER_IMAGE from its environment and falls back to its
+                # default pin when empty, so leaving DcgmExporterImage empty keeps
+                # the stock behaviour. See aws-parallelcluster-monitoring #50.
+                export DCGM_EXPORTER_IMAGE="${DcgmExporterImage}"
+                curl -fsSL https://raw.githubusercontent.com/${MonitoringRepo}/${MonitoringVersion}/post-install.sh -o /tmp/post-install.sh
+                bash /tmp/post-install.sh ${MonitoringVersion} ${MonitoringRepo} 2>&1 | tee /var/log/monitoring-install.log
+                echo "Monitoring installation complete"
+              fi
 
             --==MYBOUNDARY==
         NetworkInterfaces: