diff --git a/Makefile b/Makefile index bbbe623..2f02f31 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,14 @@ docker-run: build-whl ## Runs local Airflow for testing cd dev && astro dev restart; \ fi +.PHONY: astro-login +astro-login: # Login to Astro cloud + cd dev && astro login cloud.astronomer-stage.io + +.PHONY: deploy +deploy: build-whl astro-login ## Runs local Airflow for testing + cd dev && astro deploy -f + .PHONY: docker-stop docker-stop: ## Stop Docker container cd dev && astro dev stop diff --git a/dev/Dockerfile b/dev/Dockerfile index e26ed5e..27d8542 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -1,7 +1,21 @@ FROM quay.io/astronomer/astro-runtime:12.2.0 +ENV AIRFLOW__CORE__TEST_CONNECTION=Enabled +# You need to mount the service-account-key.json file, +# and the path should refer to the file's location inside the Docker container. +ENV GOOGLE_APPLICATION_CREDENTIALS= + USER root +RUN apt-get update -y +RUN apt-get install -y --no-install-recommends && \ + apt-get install -y curl gnupg && \ + echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - && \ + apt-get update -y && \ + apt-get install python3 google-cloud-sdk -y && \ + apt-get install google-cloud-sdk-gke-gcloud-auth-plugin -y + RUN curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 && \ chmod 700 get_helm.sh && \ ./get_helm.sh diff --git a/docs/getting_started/astro_setup.rst b/docs/getting_started/astro_setup.rst new file mode 100644 index 0000000..c0f771e --- /dev/null +++ b/docs/getting_started/astro_setup.rst @@ -0,0 +1,121 @@ +Deploy on Astro Cloud +##################### + +Deploying your project on Astro Cloud allows you to easily manage and orchestrate Ray workflows. This section guides you through the deployment process. + +Table of Contents +================= + +- `Setup GKE Cluster`_ +- `Deploy to Astro Cloud`_ +- `Troubleshoot`_ + +Setup GKE Cluster +================= + +This section describes how to create a GKE cluster that can be used to orchestrate Ray jobs using Astro Cloud. + +Prerequisites +------------- + +Install the following software: + +- `Gcloud `_ + +Steps +----- + +1. **Create a Google Cloud service account key JSON file:** Follow the instructions `here `_ + +2. **Set up the gcloud credentials** + +.. code-block:: bash + + gcloud auth activate-service-account --key-file= + + gcloud config set project + +3. **Install the gcloud auth plugin** + +.. code-block:: bash + + gcloud components install gke-gcloud-auth-plugin + +4. **Create a GKE cluster** + +.. code-block:: bash + + gcloud container clusters create \ + --zone us-central1-a \ + --num-nodes 1 \ + --machine-type e2-standard-4 \ + --enable-autoscaling --min-nodes=1 --max-nodes=3 \ + --no-enable-ip-alias + --project + +5. **Retrieve GKE cluster configuration** + +.. code-block:: bash + + gcloud container clusters get-credentials --zone us-central1-a + + kubectl config view --raw > kubeconfig.yaml + +We will use the ``kubeconfig.yaml`` file to create an Airflow connection of type ``Ray`` in Astro Cloud. You need to mount the ``kubeconfig.yaml`` file into the Airflow Docker container and specify the path to this file within the Airflow connection. + +6. ** Optional: Delete GKE cluster** + +Once you no longer need the GKE cluster, you can delete it using the command below. + +.. code-block:: bash + + gcloud container clusters delete + + +Deploy to Astro Cloud +===================== + +This section describes how to deploy the project and orchestrate Ray jobs on Astro Cloud. + +Prerequisites +------------- + +- `Docker `_ +- `Astro CLI `_ + +Steps +----- + +1. **Create an Astro deployment:** Follow the instructions `here `_ + +2. **Deploy the project on Astro Cloud** + +.. code-block:: bash + + make deploy + +This command will build a wheel from your branch and deploy the `project `_ in Astro Cloud. + +3. **Create an Airflow Connection** + +- Navigate to Admin -> Connections -> Add a new record. Select the connection type ``Ray`` and set the parameter ``Kube config path`` to the path of ``kubeconfig.yaml``. + +Troubleshoot +------------- + +1. **I'm encountering the error: "You do not currently have an active account selected for RayCluster on GKE.** + +This can occur if the environment isn't properly configured for using a service account. Please try running the command below on your machine. + +.. code-block:: bash + + gcloud auth activate-service-account --key-file='/path/to/your/service-account-key.json' + + +Alternatively, you can add a start Airflow task to execute this command. + +.. code-block:: python + + @task.bash + def activate_service_account() -> str: + return "gcloud auth activate-service-account --key-file='/path/to/your/service-account-key.json'" diff --git a/docs/index.rst b/docs/index.rst index 6345c9f..df97f61 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -9,6 +9,7 @@ Welcome to the Ray provider documentation! Home Getting started Code Samples + Deploy on Astro Cloud API Reference Contributing