[release] Add Azure VM launcher release test.

python/ray/autoscaler/azure/BUILD.bazel

@@ -9,3 +9,9 @@ filegroup(
     srcs = ["defaults.yaml"],
     visibility = ["//visibility:public"],
 )
+
+filegroup(
+    name = "test_configs",
+    data = glob(["tests/*.yaml"]),
+    visibility = ["//release:__pkg__"],
+)

python/ray/autoscaler/azure/tests/azure-cluster.yaml

@@ -1,14 +1,14 @@
 # Unique identifier for the head node and workers of this cluster.
-cluster_name: nightly-cpu-minimal
+cluster_name: nightly-cpu-minimal-centralus
 max_workers: 2
 idle_timeout_minutes: 5
 
 # Cloud-provider specific configuration.
 provider:
   type: azure
   # https://azure.microsoft.com/en-us/global-infrastructure/locations
-  location: westus2
-  resource_group: ray-nightly-cpu-minimal
+  location: centralus
+  resource_group: ray-nightly-cpu-minimal-centralus
   cache_stopped_nodes: False
 
 auth:

python/ray/autoscaler/azure/tests/azure_compute.yaml

@@ -0,0 +1,15 @@
+# This test launches an Azure VM cluster from an AWS instance.
+# The test script runs on AWS while the actual cluster is created in Azure.
+cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
+region: us-west-2
+
+head_node_type:
+    name: head_node
+    instance_type: t3.large
+
+worker_node_types:
+    - name: worker_node
+      instance_type: t3.large
+      min_workers: 0
+      max_workers: 0
+      use_spot: false

python/ray/autoscaler/launch_and_verify_cluster.py

@@ -11,6 +11,7 @@
         <cluster_configuration_file_path>
 """
 import argparse
+import json
 import os
 import re
 import subprocess
@@ -143,6 +144,51 @@ def override_docker_image(config_yaml, docker_image):
     config_yaml["docker"] = docker_config
 
 
+def azure_authenticate():
+    """Get Azure service principal credentials from AWS Secrets Manager and authenticate."""
+    print("======================================")
+    print("Getting Azure credentials from AWS Secrets Manager...")
+
+    # Initialize AWS Secrets Manager client
+    secrets_client = boto3.client("secretsmanager", region_name="us-west-2")
+
+    # Get service principal credentials
+    secret_response = secrets_client.get_secret_value(
+        SecretId="azure-service-principal-oss-release"
+    )
+    secret = secret_response["SecretString"]
+
+    client_id = json.loads(secret)["client_id"]
+    tenant_id = json.loads(secret)["tenant_id"]
+
+    # Get certificate
+    cert_response = secrets_client.get_secret_value(
+        SecretId="azure-service-principal-certificate"
+    )
+    cert = cert_response["SecretString"]
+
+    # Write certificate to temp file
+    tmp_dir = tempfile.mkdtemp()
+    cert_path = os.path.join(tmp_dir, "azure_cert.pem")
+    with open(cert_path, "w") as f:
+        f.write(cert)
+
+    # Login to Azure
+    subprocess.check_call(
+        [
+            "az",
+            "login",
+            "--service-principal",
+            "--username",
+            client_id,
+            "--certificate",
+            cert_path,
+            "--tenant",
+            tenant_id,
+        ]
+    )
+
+
 def download_ssh_key_aws():
     """Download the ssh key from the S3 bucket to the local machine."""
     print("======================================")
@@ -208,10 +254,13 @@ def cleanup_cluster(config_yaml, cluster_config):
     num_tries = 3
     for i in range(num_tries):
         try:
+            env = os.environ.copy()
+            env.pop("PYTHONPATH", None)
             subprocess.run(
                 ["ray", "down", "-v", "-y", str(cluster_config)],
                 check=True,
                 capture_output=True,
+                env=env,
             )
             cleanup_security_groups(config_yaml)
             # Final success
@@ -321,7 +370,11 @@ def cleanup_security_groups(config):
 
 
 def run_ray_commands(
-    config_yaml, cluster_config, retries, no_config_cache, num_expected_nodes=1
+    config_yaml,
+    cluster_config,
+    retries,
+    no_config_cache,
+    num_expected_nodes=1,
 ):
     """
     Run the necessary Ray commands to start a cluster, verify Ray is running, and clean
@@ -333,18 +386,19 @@ def run_ray_commands(
         no_config_cache: Whether to pass the --no-config-cache flag to the ray CLI
             commands.
     """
-
+    provider_type = config_yaml.get("provider", {}).get("type")
+    if provider_type == "azure":
+        azure_authenticate()
     print("======================================")
     print("Starting new cluster...")
     cmd = ["ray", "up", "-v", "-y"]
     if no_config_cache:
         cmd.append("--no-config-cache")
     cmd.append(str(cluster_config))
-
-    print(" ".join(cmd))
-
+    env = os.environ.copy()
+    env.pop("PYTHONPATH", None)
     try:
-        subprocess.run(cmd, check=True, capture_output=True)
+        subprocess.run(cmd, check=True, capture_output=True, env=env)
     except subprocess.CalledProcessError as e:
         print(e.output)
         # print stdout and stderr
@@ -371,7 +425,7 @@ def run_ray_commands(
             ]
             if no_config_cache:
                 cmd.append("--no-config-cache")
-            subprocess.run(cmd, check=True)
+            subprocess.run(cmd, check=True, env=env)
             success = True
             break
         except subprocess.CalledProcessError:

release/BUILD.bazel

@@ -719,6 +719,7 @@ py_test(
     ) + [
         "ray_release/tests/test_collection_data.yaml",
         "//python/ray/autoscaler/aws:test_configs",
+        "//python/ray/autoscaler/azure:test_configs",
         "//python/ray/autoscaler/gcp:test_configs",
     ],
     exec_compatible_with = ["//:hermetic_python"],

release/ray_release/byod/byod_azure_cluster_launcher.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+# This script is used to build an extra layer to run release tests on Azure.
+
+set -exo pipefail
+
+pip3 install azure-cli-core==2.21.0 azure-core azure-identity azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-common msrest msrestazure

release/release_tests.yaml

@@ -3476,6 +3476,27 @@
     prepare: bash prepare.sh
     script: python run_gcs_ft_on_k8s.py
 
+- name: azure_cluster_launcher
+  group: cluster-launcher-test
+  working_dir: ../python/ray/autoscaler/
+  frequency: nightly
+  team: clusters
+  cluster:
+    byod:
+      post_build_script: byod_azure_cluster_launcher.sh
+    cluster_compute: azure/tests/azure_compute.yaml
+  run:
+    timeout: 2400
+    script: bash release/azure_docker_login.sh && python -I launch_and_verify_cluster.py azure/tests/azure-cluster.yaml --num-expected-nodes 3 --retries 10
+
+  variations:
+    - __suffix__: v1
+      run:
+        script: RAY_UP_enable_autoscaler_v2=0 python -I launch_and_verify_cluster.py azure/tests/azure-cluster.yaml --num-expected-nodes 3 --retries 10
+    - __suffix__: v2
+      run:
+        script: RAY_UP_enable_autoscaler_v2=1 python -I launch_and_verify_cluster.py azure/tests/azure-cluster.yaml --num-expected-nodes 3 --retries 10
+
 - name: aws_cluster_launcher
   group: cluster-launcher-test
   working_dir: ../python/ray/autoscaler/