Add more re-usable actions

.github/workflows/composer-vars-update.yml

@@ -0,0 +1,68 @@
+---
+name: Composer Variables Update
+
+on:
+    workflow_call:
+        secrets:
+            WORKLOAD_ID_PROVIDER:
+                required: true
+            SERVICE_ACCOUNT:
+                required: true
+        inputs:
+            GCP_PROJECT:
+                required: true
+                type: string
+            COMPOSER_ENVIRONMENT_NAME:
+                required: true
+                type: string
+            COMPOSER_ENVIRONMENT_LOCATION:
+                required: true
+                type: string
+            VARIABLES_FILE_PATH:
+                required: true
+                type: string
+            VARIABLES_FILE_NAME:
+                required: true
+                type: string
+
+
+jobs:
+    composer-vars-update:
+        runs-on: ubuntu-latest
+        strategy:
+            max-parallel: 5
+
+        permissions:
+            contents: read
+            id-token: write
+
+        steps:
+            - uses: actions/checkout@v3
+            - name: Authenticate to Google Cloud
+              id: gcloud_auth
+              uses: 'google-github-actions/auth@v0'
+              with:
+                  workload_identity_provider: ${{ secrets.WORKLOAD_ID_PROVIDER }}
+                  service_account: ${{ secrets.SERVICE_ACCOUNT }}
+                  project_id: ${{ inputs.GCP_PROJECT }}
+                  token_format: 'access_token'
+            - name: Set up Cloud SDK
+              id: setup_cloud_sdk
+              uses: google-github-actions/setup-gcloud@v0
+            - name: Composer Storage Data Import
+              id: composer_storage_data_import
+              run: |
+                gcloud composer environments storage data import \
+                --environment=${{ inputs.COMPOSER_ENVIRONMENT_NAME }} \
+                --location=${{ inputs.COMPOSER_ENVIRONMENT_LOCATION }} \
+                --source=${{ inputs.VARIABLES_FILE_PATH }} \
+                --project=${{ inputs.GCP_PROJECT }}
+            - name: Composer Variables Import
+              id: composer_variables_import
+              run: |
+                gcloud composer environments run \
+                ${{ inputs.COMPOSER_ENVIRONMENT_NAME }} \
+                --location=${{ inputs.COMPOSER_ENVIRONMENT_LOCATION }} \
+                --project=${{ inputs.GCP_PROJECT }} \
+                variables \
+                import -- /home/airflow/gcs/data/${{ inputs.VARIABLES_FILE_NAME }}

.github/workflows/gcs-rsync.yml

@@ -0,0 +1,76 @@
+---
+name: GCS Rsync
+
+on:
+    workflow_call:
+        secrets:
+            WORKLOAD_ID_PROVIDER:
+                required: true
+            SERVICE_ACCOUNT:
+                required: true
+        inputs:
+            GCP_PROJECT:
+                required: true
+                type: string
+            SOURCE_DIR:
+                required: true
+                type: string
+            DESTINATION_DIR:
+                required: true
+                type: string
+            EXCLUDE:
+                required: false
+                type: string
+                default: ""
+            DELETE:
+                required: false
+                type: boolean
+                default: false
+            CHECKSUM_VALIDATION:
+                required: false
+                type: boolean
+                default: false
+            RECURSIVE:
+                required: false
+                type: boolean
+                default: true
+
+jobs:
+    gcs-rsync:
+        runs-on: ubuntu-latest
+        strategy:
+            max-parallel: 5
+
+        permissions:
+            contents: read
+            id-token: write
+
+        steps:
+            - uses: actions/checkout@v3
+            - name: Authenticate to Google Cloud
+              id: gcloud_auth
+              uses: 'google-github-actions/auth@v0'
+              with:
+                  workload_identity_provider: ${{ secrets.WORKLOAD_ID_PROVIDER }}
+                  service_account: ${{ secrets.SERVICE_ACCOUNT }}
+                  project_id: ${{ inputs.GCP_PROJECT }}
+                  token_format: 'access_token'
+            - name: Set up Cloud SDK
+              id: setup_cloud_sdk
+              uses: google-github-actions/setup-gcloud@v0
+            - run: |
+                echo "flags=c" >> $GITHUB_ENV
+              if: ${{ inputs.CHECKSUM_VALIDATION }}
+            - run: |
+                echo "flags=${{env.flags}}d" >> $GITHUB_ENV
+              if: ${{ inputs.DELETE }}
+            - run: |
+                echo "flags=${{env.flags}}r" >> $GITHUB_ENV
+              if: ${{ inputs.RECURSIVE }}
+            - run: |
+                echo "flags=-${{env.flags}}" >> $GITHUB_ENV
+            - run: |
+                echo "${{ env.flags }}"
+            - name: Gsutil Rsync
+              id: gsutil_rsync
+              run: gsutil -m rsync -x '${{ inputs.EXCLUDE }}' ${{ env.flags }} ${{ inputs.SOURCE_DIR }} '${{ inputs.DESTINATION_DIR }}'

README.md

@@ -4,25 +4,38 @@ This is a collection of re-usable github workflows
 
 (WIP), wait for Anders to review this before re-using! 
 
-## [build-publish-image-gcr.yml](.github/workflows/build-publish-image-gcr.yml)
-
-Pre-requisites:
+## Pre-requisites
+
 - Service Account with required permissions:
     - roles/container.developer       
     - roles/iam.serviceAccountTokenCreator    
     - roles/storage.admin      
+    - roles/composer.environmentAndStorageObjectAdmin (Composer related workflows)
+
 - [Keyless Auth Set-Up](https://cloud.google.com/blog/products/identity-security/enabling-keyless-authentication-from-github-actions)
 
+
 Once you have gone through the steps above you should have a service account and a workload id provider which you can set as secrets in your repository. 
 
-this workflow simply:
+
+
+## [build-publish-image-gcr.yml](.github/workflows/build-publish-image-gcr.yml)
+
 - reads a version file to get version number 
 - builds a docker image 
 - pushes to gcr 
 
 ## [gke-helm-upgrade.yml](.github/workflows/gke-helm-upgrade.yml)
 
-Same pre-requisites as the previous action
 
-this workflow simply:
-- takes in the helm release, chart , values , gke_cluster info to upgrade a helm deployment
+- takes in the helm release, chart , values , gke_cluster info to upgrade a helm deployment
+
+
+## [composer-vars-update.yml](.github/workflows/composer-vars-update.yml)
+
+
+- takes in a json variables file, and composer env details , updates composer variables 
+
+## [gcs-rsync.yml](.github/workflows/gcs-rsync.yml)
+
+- run gsutil rsync with some available flags