diff --git a/python/ray/autoscaler/azure/BUILD.bazel b/python/ray/autoscaler/azure/BUILD.bazel index 7d4fbc64d5de..aa4bdfd486ca 100644 --- a/python/ray/autoscaler/azure/BUILD.bazel +++ b/python/ray/autoscaler/azure/BUILD.bazel @@ -9,3 +9,9 @@ filegroup( srcs = ["defaults.yaml"], visibility = ["//visibility:public"], ) + +filegroup( + name = "test_configs", + data = glob(["tests/*.yaml"]), + visibility = ["//release:__pkg__"], +) diff --git a/python/ray/autoscaler/azure/tests/azure-cluster.yaml b/python/ray/autoscaler/azure/tests/azure-cluster.yaml index 01587c37f243..ec347fc25d16 100644 --- a/python/ray/autoscaler/azure/tests/azure-cluster.yaml +++ b/python/ray/autoscaler/azure/tests/azure-cluster.yaml @@ -1,5 +1,5 @@ # Unique identifier for the head node and workers of this cluster. -cluster_name: nightly-cpu-minimal +cluster_name: nightly-cpu-minimal-centralus max_workers: 2 idle_timeout_minutes: 5 @@ -7,8 +7,8 @@ idle_timeout_minutes: 5 provider: type: azure # https://azure.microsoft.com/en-us/global-infrastructure/locations - location: westus2 - resource_group: ray-nightly-cpu-minimal + location: centralus + resource_group: ray-nightly-cpu-minimal-centralus cache_stopped_nodes: False auth: diff --git a/python/ray/autoscaler/azure/tests/azure_compute.yaml b/python/ray/autoscaler/azure/tests/azure_compute.yaml new file mode 100644 index 000000000000..2309dbcd0977 --- /dev/null +++ b/python/ray/autoscaler/azure/tests/azure_compute.yaml @@ -0,0 +1,15 @@ +# This test launches an Azure VM cluster from an AWS instance. +# The test script runs on AWS while the actual cluster is created in Azure. +cloud_id: {{env["ANYSCALE_CLOUD_ID"]}} +region: us-west-2 + +head_node_type: + name: head_node + instance_type: t3.large + +worker_node_types: + - name: worker_node + instance_type: t3.large + min_workers: 0 + max_workers: 0 + use_spot: false diff --git a/python/ray/autoscaler/launch_and_verify_cluster.py b/python/ray/autoscaler/launch_and_verify_cluster.py index dea44b4b47d3..3c5216a18298 100644 --- a/python/ray/autoscaler/launch_and_verify_cluster.py +++ b/python/ray/autoscaler/launch_and_verify_cluster.py @@ -11,6 +11,7 @@ """ import argparse +import json import os import re import subprocess @@ -143,6 +144,51 @@ def override_docker_image(config_yaml, docker_image): config_yaml["docker"] = docker_config +def azure_authenticate(): + """Get Azure service principal credentials from AWS Secrets Manager and authenticate.""" + print("======================================") + print("Getting Azure credentials from AWS Secrets Manager...") + + # Initialize AWS Secrets Manager client + secrets_client = boto3.client("secretsmanager", region_name="us-west-2") + + # Get service principal credentials + secret_response = secrets_client.get_secret_value( + SecretId="azure-service-principal-oss-release" + ) + secret = secret_response["SecretString"] + + client_id = json.loads(secret)["client_id"] + tenant_id = json.loads(secret)["tenant_id"] + + # Get certificate + cert_response = secrets_client.get_secret_value( + SecretId="azure-service-principal-certificate" + ) + cert = cert_response["SecretString"] + + # Write certificate to temp file + tmp_dir = tempfile.mkdtemp() + cert_path = os.path.join(tmp_dir, "azure_cert.pem") + with open(cert_path, "w") as f: + f.write(cert) + + # Login to Azure + subprocess.check_call( + [ + "az", + "login", + "--service-principal", + "--username", + client_id, + "--certificate", + cert_path, + "--tenant", + tenant_id, + ] + ) + + def download_ssh_key_aws(): """Download the ssh key from the S3 bucket to the local machine.""" print("======================================") @@ -208,10 +254,13 @@ def cleanup_cluster(config_yaml, cluster_config): num_tries = 3 for i in range(num_tries): try: + env = os.environ.copy() + env.pop("PYTHONPATH", None) subprocess.run( ["ray", "down", "-v", "-y", str(cluster_config)], check=True, capture_output=True, + env=env, ) cleanup_security_groups(config_yaml) # Final success @@ -321,7 +370,11 @@ def cleanup_security_groups(config): def run_ray_commands( - config_yaml, cluster_config, retries, no_config_cache, num_expected_nodes=1 + config_yaml, + cluster_config, + retries, + no_config_cache, + num_expected_nodes=1, ): """ Run the necessary Ray commands to start a cluster, verify Ray is running, and clean @@ -333,18 +386,19 @@ def run_ray_commands( no_config_cache: Whether to pass the --no-config-cache flag to the ray CLI commands. """ - + provider_type = config_yaml.get("provider", {}).get("type") + if provider_type == "azure": + azure_authenticate() print("======================================") print("Starting new cluster...") cmd = ["ray", "up", "-v", "-y"] if no_config_cache: cmd.append("--no-config-cache") cmd.append(str(cluster_config)) - - print(" ".join(cmd)) - + env = os.environ.copy() + env.pop("PYTHONPATH", None) try: - subprocess.run(cmd, check=True, capture_output=True) + subprocess.run(cmd, check=True, capture_output=True, env=env) except subprocess.CalledProcessError as e: print(e.output) # print stdout and stderr @@ -371,7 +425,7 @@ def run_ray_commands( ] if no_config_cache: cmd.append("--no-config-cache") - subprocess.run(cmd, check=True) + subprocess.run(cmd, check=True, env=env) success = True break except subprocess.CalledProcessError: diff --git a/release/BUILD.bazel b/release/BUILD.bazel index 5ab632c401f8..63edf962da7b 100644 --- a/release/BUILD.bazel +++ b/release/BUILD.bazel @@ -719,6 +719,7 @@ py_test( ) + [ "ray_release/tests/test_collection_data.yaml", "//python/ray/autoscaler/aws:test_configs", + "//python/ray/autoscaler/azure:test_configs", "//python/ray/autoscaler/gcp:test_configs", ], exec_compatible_with = ["//:hermetic_python"], diff --git a/release/ray_release/byod/byod_azure_cluster_launcher.sh b/release/ray_release/byod/byod_azure_cluster_launcher.sh new file mode 100755 index 000000000000..052d543182fc --- /dev/null +++ b/release/ray_release/byod/byod_azure_cluster_launcher.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# This script is used to build an extra layer to run release tests on Azure. + +set -exo pipefail + +pip3 install azure-cli-core==2.21.0 azure-core azure-identity azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-common msrest msrestazure diff --git a/release/release_tests.yaml b/release/release_tests.yaml index 363f355b3c3e..c1762b7d056c 100644 --- a/release/release_tests.yaml +++ b/release/release_tests.yaml @@ -3476,6 +3476,27 @@ prepare: bash prepare.sh script: python run_gcs_ft_on_k8s.py +- name: azure_cluster_launcher + group: cluster-launcher-test + working_dir: ../python/ray/autoscaler/ + frequency: nightly + team: clusters + cluster: + byod: + post_build_script: byod_azure_cluster_launcher.sh + cluster_compute: azure/tests/azure_compute.yaml + run: + timeout: 2400 + script: bash release/azure_docker_login.sh && python -I launch_and_verify_cluster.py azure/tests/azure-cluster.yaml --num-expected-nodes 3 --retries 10 + + variations: + - __suffix__: v1 + run: + script: RAY_UP_enable_autoscaler_v2=0 python -I launch_and_verify_cluster.py azure/tests/azure-cluster.yaml --num-expected-nodes 3 --retries 10 + - __suffix__: v2 + run: + script: RAY_UP_enable_autoscaler_v2=1 python -I launch_and_verify_cluster.py azure/tests/azure-cluster.yaml --num-expected-nodes 3 --retries 10 + - name: aws_cluster_launcher group: cluster-launcher-test working_dir: ../python/ray/autoscaler/