From 893eb7b51d3d59b60afa0390ed61579948ba5480 Mon Sep 17 00:00:00 2001 From: pankajastro Date: Fri, 1 Nov 2024 00:27:22 +0530 Subject: [PATCH 1/3] [Docs] Add docs to deploy project on Astro Cloud Add documentation for deploying the project in the /dev directory on Astro Cloud. I created a Makefile target called "deploy" for this purpose. For this exercise, I set up an Astro deployment on AWS Cloud and a RayCluster on GKE. To deploy on Astro Cloud, run the command make deploy. closes: https://github.com/astronomer/astro-provider-ray/issues/88 --- Makefile | 8 ++ dev/Dockerfile.ray_google_cloud | 22 +++++ docs/getting_started/astro_setup.rst | 115 +++++++++++++++++++++++++++ docs/index.rst | 1 + 4 files changed, 146 insertions(+) create mode 100644 dev/Dockerfile.ray_google_cloud create mode 100644 docs/getting_started/astro_setup.rst diff --git a/Makefile b/Makefile index bbbe623..50289b5 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,14 @@ docker-run: build-whl ## Runs local Airflow for testing cd dev && astro dev restart; \ fi +.PHONY: astro-login +astro-login: # Login to Astro cloud + cd dev && astro login + +.PHONY: deploy +deploy: build-whl astro-login ## Runs local Airflow for testing + cd dev && astro deploy -f + .PHONY: docker-stop docker-stop: ## Stop Docker container cd dev && astro dev stop diff --git a/dev/Dockerfile.ray_google_cloud b/dev/Dockerfile.ray_google_cloud new file mode 100644 index 0000000..e97dd72 --- /dev/null +++ b/dev/Dockerfile.ray_google_cloud @@ -0,0 +1,22 @@ +FROM quay.io/astronomer/astro-runtime:12.2.0 + +ENV GOOGLE_APPLICATION_CREDENTIALS=/path/to/your/service-account-key.json + +USER root + +RUN apt-get update -y +RUN apt-get install -y --no-install-recommends && \ + apt-get install -y curl gnupg && \ + echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - && \ + apt-get update -y && \ + apt-get install python3 google-cloud-sdk -y && \ + apt-get install google-cloud-sdk-gke-gcloud-auth-plugin -y + +RUN curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 && \ + chmod 700 get_helm.sh && \ + ./get_helm.sh + +USER astro + +RUN pip install /usr/local/airflow/include/*.whl diff --git a/docs/getting_started/astro_setup.rst b/docs/getting_started/astro_setup.rst new file mode 100644 index 0000000..4f4b7c4 --- /dev/null +++ b/docs/getting_started/astro_setup.rst @@ -0,0 +1,115 @@ +Deploy on Astro Cloud +##################### + +Deploying your project on Astro Cloud allows you to easily manage and orchestrate Ray workflows. This section guides you through the deployment process. + +Table of Contents +================= + +- `Setup GKE Cluster`_ +- `Deploy to Astro Cloud`_ +- `Troubleshoot`_ + +Setup GKE Cluster +================= + +This section describes how to create a GKE cluster that can be used to orchestrate Ray jobs using Astro Cloud. + +Prerequisites +------------- + +Install the following software: + +- `Gcloud `_ + +Steps +----- + +1. **Create a Google Cloud service account key JSON file:** Follow the instructions `here `_ + +2. **Set up the gcloud credentials** + +.. code-block:: bash + + export GOOGLE_APPLICATION_CREDENTIALS=/path/to/your/service-account-key.json + + gcloud auth activate-service-account --key-file=/path/to/your/service-account-key.json + +3. **Install the gcloud auth plugin** + +.. code-block:: bash + + gcloud components install gke-gcloud-auth-plugin + +4. **Create a GKE cluster** + +.. code-block:: bash + + gcloud container clusters create cluster-name \ + --zone us-central1-a \ + --num-nodes 1 \ + --machine-type e2-standard-4 \ + --enable-autoscaling --min-nodes=1 --max-nodes=3 \ + --no-enable-ip-alias + +5. **Retrieve GKE cluster configuration** + +.. code-block:: bash + + gcloud container clusters get-credentials cluster-name --zone us-central1-a + + kubectl config view --raw > kubeconfig.yaml + +We will use this ``kubeconfig.yaml`` to create an Airflow connection type ``Ray`` in Astro Cloud. + + +Deploy to Astro Cloud +===================== + +This section describes how to deploy the project and orchestrate Ray jobs on Astro Cloud. + +Prerequisites +------------- + +- `Docker `_ +- `Astro CLI `_ + +Steps +----- + +1. **Create an Astro deployment:** Follow the instructions `here `_ + +2. **Deploy the project on Astro Cloud** + + +Please ensure you use `Dockerfile.ray_google_cloud `_ as the default. The command below uses `Dockerfile `_ by default. + +.. code-block:: bash + + make deploy + +This command will build a wheel from your branch and deploy the `project `_ in Astro Cloud + +3. **Create an Airflow Connection** + +- Navigate to Admin -> Connections -> Add a new record. Select the connection type ``Ray`` and set the parameter ``Kube config path`` to the path of ``kubeconfig.yaml``. + +Troubleshoot +------------- + +1. **I'm encountering the error: "You do not currently have an active account selected for RayCluster on GKE.** + +This can occur if the environment isn't properly configured for using a service account. Please try running the command below on your machine. + +.. code-block:: bash + + gcloud auth activate-service-account --key-file='/path/to/your/service-account-key.json' + + +Alternatively, you can add a starting Airflow task to execute this command. + +.. code-block:: python + + @task.bash + def activate_service_account() -> str: + return "gcloud auth activate-service-account --key-file='/path/to/your/service-account-key.json'" diff --git a/docs/index.rst b/docs/index.rst index 6345c9f..df97f61 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -9,6 +9,7 @@ Welcome to the Ray provider documentation! Home Getting started Code Samples + Deploy on Astro Cloud API Reference Contributing From c4a9d2a015e78391d7d1de53dfbd0a99e95fc264 Mon Sep 17 00:00:00 2001 From: pankajastro Date: Wed, 6 Nov 2024 00:19:45 +0530 Subject: [PATCH 2/3] Apply review suggestions --- Makefile | 2 +- dev/Dockerfile | 12 ++++++++++++ dev/Dockerfile.ray_google_cloud | 22 ---------------------- docs/getting_started/astro_setup.rst | 24 +++++++++++++++--------- 4 files changed, 28 insertions(+), 32 deletions(-) delete mode 100644 dev/Dockerfile.ray_google_cloud diff --git a/Makefile b/Makefile index 50289b5..2f02f31 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ docker-run: build-whl ## Runs local Airflow for testing .PHONY: astro-login astro-login: # Login to Astro cloud - cd dev && astro login + cd dev && astro login cloud.astronomer-stage.io .PHONY: deploy deploy: build-whl astro-login ## Runs local Airflow for testing diff --git a/dev/Dockerfile b/dev/Dockerfile index e26ed5e..767b836 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -1,7 +1,19 @@ FROM quay.io/astronomer/astro-runtime:12.2.0 +ENV AIRFLOW__CORE__TEST_CONNECTION=Enabled +ENV GOOGLE_APPLICATION_CREDENTIALS= + USER root +RUN apt-get update -y +RUN apt-get install -y --no-install-recommends && \ + apt-get install -y curl gnupg && \ + echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - && \ + apt-get update -y && \ + apt-get install python3 google-cloud-sdk -y && \ + apt-get install google-cloud-sdk-gke-gcloud-auth-plugin -y + RUN curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 && \ chmod 700 get_helm.sh && \ ./get_helm.sh diff --git a/dev/Dockerfile.ray_google_cloud b/dev/Dockerfile.ray_google_cloud deleted file mode 100644 index e97dd72..0000000 --- a/dev/Dockerfile.ray_google_cloud +++ /dev/null @@ -1,22 +0,0 @@ -FROM quay.io/astronomer/astro-runtime:12.2.0 - -ENV GOOGLE_APPLICATION_CREDENTIALS=/path/to/your/service-account-key.json - -USER root - -RUN apt-get update -y -RUN apt-get install -y --no-install-recommends && \ - apt-get install -y curl gnupg && \ - echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - && \ - apt-get update -y && \ - apt-get install python3 google-cloud-sdk -y && \ - apt-get install google-cloud-sdk-gke-gcloud-auth-plugin -y - -RUN curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 && \ - chmod 700 get_helm.sh && \ - ./get_helm.sh - -USER astro - -RUN pip install /usr/local/airflow/include/*.whl diff --git a/docs/getting_started/astro_setup.rst b/docs/getting_started/astro_setup.rst index 4f4b7c4..ff15de9 100644 --- a/docs/getting_started/astro_setup.rst +++ b/docs/getting_started/astro_setup.rst @@ -31,9 +31,9 @@ Steps .. code-block:: bash - export GOOGLE_APPLICATION_CREDENTIALS=/path/to/your/service-account-key.json + gcloud auth activate-service-account --key-file= - gcloud auth activate-service-account --key-file=/path/to/your/service-account-key.json + gcloud config set project 3. **Install the gcloud auth plugin** @@ -45,23 +45,32 @@ Steps .. code-block:: bash - gcloud container clusters create cluster-name \ + gcloud container clusters create \ --zone us-central1-a \ --num-nodes 1 \ --machine-type e2-standard-4 \ --enable-autoscaling --min-nodes=1 --max-nodes=3 \ --no-enable-ip-alias + --project 5. **Retrieve GKE cluster configuration** .. code-block:: bash - gcloud container clusters get-credentials cluster-name --zone us-central1-a + gcloud container clusters get-credentials --zone us-central1-a kubectl config view --raw > kubeconfig.yaml We will use this ``kubeconfig.yaml`` to create an Airflow connection type ``Ray`` in Astro Cloud. +6. ** Optional: Delete GKE cluster** + +Once you no longer need the GKE cluster, you can delete it using the command below. + +.. code-block:: bash + + gcloud container clusters delete + Deploy to Astro Cloud ===================== @@ -81,14 +90,11 @@ Steps 2. **Deploy the project on Astro Cloud** - -Please ensure you use `Dockerfile.ray_google_cloud `_ as the default. The command below uses `Dockerfile `_ by default. - .. code-block:: bash make deploy -This command will build a wheel from your branch and deploy the `project `_ in Astro Cloud +This command will build a wheel from your branch and deploy the `project `_ in Astro Cloud. 3. **Create an Airflow Connection** @@ -106,7 +112,7 @@ This can occur if the environment isn't properly configured for using a service gcloud auth activate-service-account --key-file='/path/to/your/service-account-key.json' -Alternatively, you can add a starting Airflow task to execute this command. +Alternatively, you can add a start Airflow task to execute this command. .. code-block:: python From eca8821105f6b191d095445ec48316230d9b7a09 Mon Sep 17 00:00:00 2001 From: pankajastro Date: Wed, 6 Nov 2024 00:32:58 +0530 Subject: [PATCH 3/3] Apply review suggestion --- dev/Dockerfile | 2 ++ docs/getting_started/astro_setup.rst | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dev/Dockerfile b/dev/Dockerfile index 767b836..27d8542 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -1,6 +1,8 @@ FROM quay.io/astronomer/astro-runtime:12.2.0 ENV AIRFLOW__CORE__TEST_CONNECTION=Enabled +# You need to mount the service-account-key.json file, +# and the path should refer to the file's location inside the Docker container. ENV GOOGLE_APPLICATION_CREDENTIALS= USER root diff --git a/docs/getting_started/astro_setup.rst b/docs/getting_started/astro_setup.rst index ff15de9..c0f771e 100644 --- a/docs/getting_started/astro_setup.rst +++ b/docs/getting_started/astro_setup.rst @@ -61,7 +61,7 @@ Steps kubectl config view --raw > kubeconfig.yaml -We will use this ``kubeconfig.yaml`` to create an Airflow connection type ``Ray`` in Astro Cloud. +We will use the ``kubeconfig.yaml`` file to create an Airflow connection of type ``Ray`` in Astro Cloud. You need to mount the ``kubeconfig.yaml`` file into the Airflow Docker container and specify the path to this file within the Airflow connection. 6. ** Optional: Delete GKE cluster**