Google Images #518
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Google Images | |
| on: | |
| schedule: | |
| - cron: "45 2 * * *" | |
| # push: | |
| # branches: | |
| # - main | |
| workflow_dispatch: | |
| inputs: | |
| environment: | |
| description: 'Choose your deployment target environment' | |
| required: true | |
| default: 'staging' | |
| type: choice | |
| options: | |
| - 'production' | |
| - 'staging' | |
| permissions: | |
| actions: write | |
| contents: read | |
| id-token: write | |
| env: | |
| AWS_DEFAULT_OUTPUT: json | |
| AWS_PAGER: "" | |
| jobs: | |
| setup: | |
| name: Setup workflow envs | |
| runs-on: ubuntu-latest | |
| outputs: | |
| environment: ${{ steps.set-environment.outputs.environment }} | |
| bucket: ${{ steps.set-bucket.outputs.bucket }} | |
| steps: | |
| # Only set target environment to production if the workflow is triggered by | |
| # a scheduled execution or workflow_dispatch with production input selected | |
| - name: Setup target environment | |
| id: set-environment | |
| run: | | |
| if [ "${{ github.event_name }}" = "schedule" ] || [ "${{ inputs.environment }}" = "production" ]; then | |
| echo "environment=production" >> $GITHUB_OUTPUT | |
| else | |
| echo "environment=staging" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Setup target bucket | |
| # Set target bucket according to target environment | |
| id: set-bucket | |
| run: | | |
| if [ "${{ github.event_name }}" = "schedule" ] || [ "${{ inputs.environment }}" = "production" ]; then | |
| echo "bucket=cloudx-json-bucket" >> $GITHUB_OUTPUT | |
| else | |
| echo "bucket=cid-bucket-staging" >> $GITHUB_OUTPUT | |
| fi | |
| list_images: | |
| name: "List Images" | |
| runs-on: ubuntu-latest | |
| needs: setup | |
| env: | |
| REGION: ${{ matrix.region }} | |
| ENVIRONMENT: ${{ needs.setup.outputs.environment }} | |
| steps: | |
| - name: Clone repo | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Checkout latest tagged release | |
| if: env.ENVIRONMENT == 'production' | |
| run: | | |
| LATEST_RELEASE=$(git describe --tags `git rev-list --tags --max-count=1`) | |
| git checkout $LATEST_RELEASE | |
| - id: "auth" | |
| name: "Authenticate to Google Cloud" | |
| uses: "google-github-actions/auth@v2" | |
| with: | |
| workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }} | |
| - name: "Set up Cloud SDK" | |
| uses: "google-github-actions/setup-gcloud@v2" | |
| - name: Get images | |
| run: | | |
| gcloud compute images list --project=rhel-cloud --no-standard-images --format="json" | jq -c > global.json | |
| zstd -v global.json | |
| - name: Store image data in artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: "google" | |
| path: "global.json.zst" | |
| upload_data: | |
| name: "Upload Data" | |
| runs-on: ubuntu-latest | |
| needs: [list_images, setup] | |
| env: | |
| ENVIRONMENT: ${{ needs.setup.outputs.environment }} | |
| BUCKET: ${{ needs.setup.outputs.bucket }} | |
| steps: | |
| - name: Clone repo | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Checkout latest tagged release | |
| if: env.ENVIRONMENT == 'production' | |
| run: | | |
| LATEST_RELEASE=$(git describe --tags `git rev-list --tags --max-count=1`) | |
| git checkout $LATEST_RELEASE | |
| - name: Download image data artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: raw | |
| - name: Install awscli v2 | |
| uses: unfor19/install-aws-cli-action@v1 | |
| - name: Configure AWS credentials for production | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| if: env.ENVIRONMENT == 'production' | |
| with: | |
| role-to-assume: arn:aws:iam::426579533370:role/github_actions_image_retriever | |
| role-duration-seconds: 1800 | |
| aws-region: us-east-1 | |
| - name: Configure AWS credentials for staging | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| if: env.ENVIRONMENT == 'staging' | |
| with: | |
| role-to-assume: arn:aws:iam::426579533370:role/github_actions_cloud_image_directory_staging | |
| role-duration-seconds: 1800 | |
| aws-region: us-east-1 | |
| - name: Decompress artifacts | |
| run: | | |
| pushd raw/google | |
| zstd -vd --rm * | |
| popd | |
| - name: Upload raw data to S3 | |
| run: | | |
| aws s3 sync \ | |
| --acl public-read \ | |
| --delete \ | |
| --no-progress \ | |
| --content-type application/json \ | |
| $(pwd)/raw/google/ s3://${BUCKET}/raw/google/ |