yosefe · May 11, 2025 · Apr 27, 2025 · Apr 27, 2025 · Apr 27, 2025 · Apr 30, 2025
@@ -0,0 +1,69 @@
+name: AWS EFA NIXL Validation
+
+on:
+  push:
+    branches:
+    - main
+    - "pull-request/[0-9]+"
+
+jobs:
+  run_aws_tests:
+    name: Run AWS Tests - ${{ matrix.test_name }}
+    environment: SWX_AWS
+    runs-on: ubuntu-latest
+    env:
+      AWS_DEFAULT_REGION: eu-central-1
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - test_name: "C++ Tests"
+            test_scripts:
+              - .gitlab/test_cpp.sh $NIXL_INSTALL_DIR
+          - test_name: "Python Tests"
+            test_scripts:
+              - .gitlab/test_python.sh $NIXL_INSTALL_DIR
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Setup
+        run: |
+          set -exE
+          sudo apt-get -qq update
+          sudo apt-get -qq install -y curl gettext git gzip jq tar unzip
+
+          # Install kubectl
+          curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+
+          # Install AWS CLI
+          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+          unzip -qq awscliv2.zip
+          sudo ./aws/install --update
+
+          # Verify AWS credentials
+          aws sts get-caller-identity >/dev/null
+
+      - name: Run AWS tests
+        working-directory: ./contrib/aws-efa
+        timeout-minutes: 180
+        run: |
+          set -o pipefail
+          test_cmd='${{ join(matrix.test_scripts, ' && ') }}'
+          ./aws_test.sh "$test_cmd" 2>&1 | tee test_output.log
+
+      - name: Cleanup AWS resources
+        if: always()
+        working-directory: ./contrib/aws-efa
+        run: |
+          JOB_ID=$(grep 'JOB_ID=' test_output.log | head -n1 | cut -d= -f2)
+          echo "JOB_ID=$JOB_ID"
+          aws batch terminate-job \
+            --job-id "$JOB_ID" \
+            --reason 'Terminated by GitHub Actions' || true
@@ -145,7 +145,7 @@ $global:copyright_results = @{
 
 # === end common.ps1 extensions ===
 
-$ignored_files = @('.clang-format', '.gitattributes', '.gitignore', '.gitkeep', '.patch', 'Cargo.lock', 'LICENSE', 'uv.lock', 'rust-toolchain.toml', 'codespell.txt')
+$ignored_files = @('.clang-format', '.gitattributes', '.gitignore', '.gitkeep', '.patch', 'Cargo.lock', 'LICENSE', 'uv.lock', 'rust-toolchain.toml', 'codespell.txt', 'aws_job_def.json')
 write-debug "<copyright-check> ignored_files = ['$($ignored_files -join "','")']."
 $ignored_paths = @('.github', '.mypy_cache', '.pytest_cache')
 write-debug "<copyright-check> ignored_paths = ['$($ignored_paths -join "','")']."

@@ -5,4 +5,7 @@
 subprojects/abseil-cpp-*
 subprojects/packagecache/
 
-.vscode
+.vscode
+
+# AWS EFA test files
+contrib/aws-efa/aws_vars.json
@@ -0,0 +1,73 @@
+# NIXL AWS Testing
+
+This directory contains scripts and configuration files for running NIXL tests on AWS infrastructure using AWS Batch and EKS.
+
+## Overview
+
+The AWS test infrastructure allows NIXL to be automatically tested on AWS Elastic Fabric Adapter (EFA) environments through GitHub Actions. It leverages AWS Batch to manage compute resources and job execution.
+
+## Prerequisites
+
+- AWS account with access to EKS and AWS Batch
+- Pre-configured AWS job queue: `ucx-nxil-jq`
+- Pre-configured AWS EKS cluster: `ucx-ci`
+- Properly registered job definition: `NIXL-Ubuntu-JD`
+
+## Files
+
+- **aws_test.sh**: Main script that submits and monitors AWS Batch jobs
+- **aws_vars.template**: Template file for AWS Batch job configuration
+- **aws_job_def.json**: Job definition (Registered once, for reference only)
+
+## GitHub Actions Integration
+
+The script is designed to run in GitHub Actions when branches are updated.
+It relies on the `GITHUB_REF` environment variable to determine which branch or commit to test. The value can be a branch name (e.g., `main`, `feature-xyz`) or a commit SHA.
+
+### Required GitHub Secrets
+
+The following secrets are configured in the GitHub environment:
+
+- `AWS_ACCESS_KEY_ID`: AWS access key with permissions for AWS Batch and EKS
+- `AWS_SECRET_ACCESS_KEY`: Corresponding secret access key
+
+## Manual Execution
+
+To run the tests manually:
+
+1. Set your AWS account using `aws configure` command or env vars.
+2. Substitute GH variables:
+
+```bash
+# Branch or commit SHA to test
+export GITHUB_REF="main"
+
+# Other required variables
+export GITHUB_SERVER_URL="https://github.com"
+export GITHUB_REPOSITORY="ai-dynamo/nixl"
+
+# Run the script with your test command(s)
+./aws_test.sh ".gitlab/test_cpp.sh /opt/nixl"
+
+# Multiple test commands can be chained with '&&'
+./aws_test.sh ".gitlab/test_cpp.sh /opt/nixl && .test_script2.sh param123"
+```
+
+## Test Execution Flow
+
+The AWS test script:
+
+1. Generates AWS job configuration from template
+2. Submits a job to AWS
+3. Monitors AWS job execution
+4. Streams logs from the AWS pod
+5. Reports success or failure
+
+## Container Image
+
+The script uses the container image: `nvcr.io/nvidia/pytorch:25.02-py3`
+You can override this by setting the `CONTAINER_IMAGE` environment variable:
+
+```bash
+export CONTAINER_IMAGE="your-custom-image:tag"
+```
@@ -0,0 +1,54 @@
+{
+    "jobDefinitionName": "NIXL-Ubuntu-JD",
+    "type": "container",
+    "parameters": {},
+    "timeout": {
+        "attemptDurationSeconds": 10800
+    },
+    "tags": {
+        "Project": "NIXL",
+        "Environment": "Testing"
+    },
+    "eksProperties": {
+        "podProperties": {
+            "hostNetwork": true,
+            "imagePullSecrets": [],
+            "containers": [
+                {
+                    "image": "nvcr.io/nvidia/pytorch:25.02-py3",
+                    "command": [
+                        "/bin/bash",
+                        "-c",
+                        "sleep 1h"
+                    ],
+                    "args": [],
+                    "env": [],
+                    "resources": {
+                        "limits": {
+                            "memory": "167936Mi",
+                            "cpu": "70"
+                        }
+                    },
+                    "volumeMounts": [
+                        {
+                            "name": "efa-volume",
+                            "mountPath": "/dev/infiniband/uverbs0"
+                        }
+                    ],
+                    "securityContext": {
+                        "privileged": true
+                    }
+                }
+            ],
+            "initContainers": [],
+            "volumes": [
+                {
+                    "name": "efa-volume",
+                    "hostPath": {
+                        "path": "/dev/infiniband/uverbs0"
+                    }
+                }
+            ]
+        }
+    }
+}
@@ -0,0 +1,125 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -exE -o pipefail
+
+usage() {
+    echo ""
+    echo "Description: Run NIXL tests on AWS infrastructure"
+    echo ""
+    echo "Usage: $0 <test script> <test script args>"
+    echo ""
+    echo "Example: $0 .gitlab/test_cpp.sh \$NIXL_INSTALL_DIR && .test_script123.sh param123"
+    echo ""
+    echo "Required environment variables:"
+    echo "  GITHUB_REF        - Git reference to checkout (e.g., main, branch-xyz, commit SHA)"
+    echo "  GITHUB_SERVER_URL - GitHub server URL (e.g., \"https://github.com\")"
+    echo "  GITHUB_REPOSITORY - GitHub repository (e.g., \"ai-dynamo/nixl\")"
+    echo ""
+    echo "Optional environment variables:"
+    echo "  CONTAINER_IMAGE   - Container image to use (default: nvcr.io/nvidia/pytorch:25.02-py3)"
+    exit 1
+}
+
+# Validate required parameters and environment variables
+if [ -z "$1" ]; then
+    echo "Error: Test command string argument is required"
+    usage
+fi
+
+if [ -z "$GITHUB_REF" ] || [ -z "$GITHUB_SERVER_URL" ] || [ -z "$GITHUB_REPOSITORY" ]; then
+    echo "Error: Missing required environment variables"
+    usage
+fi
+
+test_cmd="$1"
+export CONTAINER_IMAGE=${CONTAINER_IMAGE:-"nvcr.io/nvidia/pytorch:25.02-py3"}
+
+# Set Git checkout command based on GITHUB_REF
+case "$GITHUB_REF" in
+    refs/heads/*)
+        export GIT_CHECKOUT_CMD="git checkout ${GITHUB_REF#refs/heads/}"
+        ;;
+    *)
+        export GIT_CHECKOUT_CMD="git checkout $GITHUB_REF"
+        ;;
+esac
+
+# Construct command sequence to run within the AWS container
+setup_cmd="set -x && \
+    git clone ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY} && \
+    cd nixl && \
+    ${GIT_CHECKOUT_CMD}"
+build_cmd=".gitlab/build.sh \${NIXL_INSTALL_DIR} \${UCX_INSTALL_DIR}"
+export AWS_CMD="${setup_cmd} && ${build_cmd} && ${test_cmd}"
+
+# Generate AWS job properties json from template
+envsubst < aws_vars.template > aws_vars.json
+jq . aws_vars.json >/dev/null
+
+# Submit AWS job
+aws eks update-kubeconfig --name ucx-ci
+JOB_NAME="NIXL_${GITHUB_RUN_NUMBER:-$RANDOM}"
+JOB_ID=$(aws batch submit-job \
+    --job-name "$JOB_NAME" \
+    --job-definition "NIXL-Ubuntu-JD" \
+    --job-queue ucx-nxil-jq \
+    --eks-properties-override file://./aws_vars.json \
+    --query 'jobId' --output text)
+
+# Function to wait for a specific job status
+wait_for_status() {
+    local target_status="$1"
+    local timeout=600
+    local interval=60
+    local status=""
+    SECONDS=0
+
+    while [ $SECONDS -lt $timeout ]; do
+        status=$(aws batch describe-jobs --jobs "$JOB_ID" --query 'jobs[0].status' --output text)
+        echo "Current status: $status (${SECONDS}s elapsed)"
+        if echo "$status" | grep -qE "$target_status"; then
+            echo "Reached status $status (completed in ${SECONDS}s)"
+            return 0
+        fi
+        sleep $interval
+    done
+
+    echo "Timeout waiting for status $target_status after ${SECONDS}s. Final status: $status"
+    return 1
+}
+
+# Wait for the job to start running
+echo "Waiting for job to start running..."
+if ! wait_for_status "RUNNING"; then
+    echo "Job failed to start"
+    exit 1
+fi
+
+# Stream logs from the pod
+POD=$(aws batch describe-jobs --jobs "$JOB_ID" --query 'jobs[0].eksProperties.podProperties.podName' --output text)
+echo "Streaming logs from pod: $POD"
+kubectl -n ucx-ci-batch-nodes logs -f "$POD"
+
+# Check final job status
+echo "Waiting for job completion..."
+exit_status=$(wait_for_status "SUCCEEDED|FAILED")
+if [[ "$exit_status" =~ FAILED ]]; then
+    echo "Failure running NIXL tests"
+    exit 1
+fi
+
+echo "NIXL tests completed successfully"
@@ -0,0 +1,36 @@
+{
+    "podProperties": {
+        "containers": [
+            {
+                "command": [
+                    "/bin/bash",
+                    "-c",
+                    "${AWS_CMD}"
+                ],
+                "image": "${CONTAINER_IMAGE}",
+                "env": [
+                    {
+                        "name": "UCX_INSTALL_DIR",
+                        "value": "/usr/local"
+                    },
+                    {
+                        "name": "NIXL_INSTALL_DIR",
+                        "value": "/opt/nixl"
+                    },
+                    {
+                        "name": "LDFLAGS",
+                        "value": "-lpthread -ldl"
+                    },
+                    {
+                        "name": "LD_LIBRARY_PATH",
+                        "value": "/opt/nixl/lib:/opt/nixl/lib/x86_64-linux-gnu:/opt/nixl/lib/x86_64-linux-gnu/plugins:/usr/local/lib"
+                    },
+                    {
+                        "name": "NIXL_PLUGIN_DIR",
+                        "value": "/opt/nixl/lib/x86_64-linux-gnu/plugins"
+                    }
+                ]
+            }
+        ]
+    }
+}