end2end.yaml

---
name: build-iso-and-end2end-test

concurrency:
  group: "zenko-tests-${{ github.ref_name }}"
  cancel-in-progress: true

on:
  push:
    branches-ignore:
      - 'development/**'
      - 'q/*'

env:
  # kind-env
  WORKER_COUNT: '2'
  OPERATOR_REPO: git@github.com:scality/zenko-operator.git
  OPERATOR_IMAGE: ""
  KIND_NODE_IMAGE: "kindest/node:v1.23.4@sha256:0e34f0d0fd448aa2f2819cfd74e99fe5793a6e4938b328f657c8e3f81ee0dfb9"
  VOLUME_ROOT: /artifacts
  OIDC_REALM: "zenko"
  OIDC_CLIENT_ID: "zenko-ui"
  OIDC_USERNAME: 'storage_manager'
  OIDC_PASSWORD: '123'
  OIDC_FIRST_NAME: 'hello'
  OIDC_LAST_NAME: 'world'
  SHELL_UI_NAME: "shell-ui"
  SHELL_UI_IMAGE: "ghcr.io/scality/metalk8s/shell-ui:v127.0.0"
  HTTP_PROXY: ""
  HTTPS_PROXY: ""
  BACKBEAT_BUCKET_CHECK_TIMEOUT_S: ""
  BACKBEAT_LCC_CRON_RULE: ""
  # e2e-env
  E2E_IMAGE_NAME: ghcr.io/scality/zenko/zenko-e2e
  E2E_CTST_IMAGE_NAME: ghcr.io/scality/zenko/zenko-e2e-ctst
  E2E_IMAGE_TAG: ${{ github.sha }}
  VAULT_TEST_IMAGE_NAME: ""
  VAULT_TEST_IMAGE_TAG: ""
  # http-env
  UI_ENDPOINT: 'http://ui.zenko.local'
  OIDC_ENDPOINT: 'http://keycloak.zenko.local'
  OIDC_HOST: 'keycloak.zenko.local'
  NAVBAR_ENDPOINT: 'http://shell-ui.zenko.local'
  ENABLE_KEYCLOAK_HTTPS: 'false'
  PROMETHEUS_NAME: "prometheus"
  # mock-env
  AZURE_ACCOUNT_NAME: devstoreaccount1
  AZURE_BACKEND_ENDPOINT: http://devstoreaccount1.blob.azure-mock.zenko.local
  AZURE_BACKEND_QUEUE_ENDPOINT: http://devstoreaccount1.queue.azure-mock.zenko.local
  AZURE_SECRET_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
  AWS_ENDPOINT: http://aws-mock.zenko.local
  AWS_ACCESS_KEY: accessKey1
  AWS_SECRET_KEY: verySecretKey1
  VERIFY_CERTIFICATES: "false"
  # secrets-env
  AWS_BACKEND_SOURCE_LOCATION: awsbackend
  AWS_BACKEND_DESTINATION_LOCATION: awsbackendmismatch
  AWS_BACKEND_DESTINATION_FAIL_LOCATION: awsbackendfail
  GCP_BACKEND_DESTINATION_LOCATION: gcpbackendmismatch
  AZURE_BACKEND_DESTINATION_LOCATION: azurebackendmismatch
  COLD_BACKEND_DESTINATION_LOCATION: e2e-cold
  AZURE_ARCHIVE_BACKEND_DESTINATION_LOCATION: e2e-azure-archive
  LOCATION_QUOTA_BACKEND: quotabackend
  AWS_BUCKET_NAME: ci-zenko-aws-target-bucket
  AWS_CRR_BUCKET_NAME: ci-zenko-aws-crr-target-bucket
  AWS_FAIL_BUCKET_NAME: ci-zenko-aws-fail-target-bucket
  AZURE_CRR_BUCKET_NAME: ci-zenko-azure-crr-target-bucket
  AZURE_ARCHIVE_BUCKET_NAME: ci-zenko-azure-archive-target-bucket
  AZURE_ARCHIVE_BUCKET_NAME_2: ci-zenko-azure-archive-target-bucket-2
  AZURE_ARCHIVE_QUEUE_NAME: ci-zenko-azure-archive-target-queue
  GCP_CRR_BUCKET_NAME: ci-zenko-gcp-crr-target-bucket
  GCP_CRR_MPU_BUCKET_NAME: ci-zenko-gcp-crr-mpu-bucket
  GCP_ACCESS_KEY: ${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY }}
  GCP_SECRET_KEY: ${{ secrets.AWS_GCP_BACKEND_SECRET_KEY }}
  GCP_BACKEND_SERVICE_KEY: ${{ secrets.GCP_BACKEND_SERVICE_KEY }}
  GCP_BACKEND_SERVICE_EMAIL: ${{ secrets.GCP_BACKEND_SERVICE_EMAIL }}
  # Enable this for Ring tests
  ENABLE_RING_TESTS: "false"
  RING_S3C_ACCESS_KEY: ${{ secrets.RING_S3C_BACKEND_ACCESS_KEY }}
  RING_S3C_SECRET_KEY: ${{ secrets.RING_S3C_BACKEND_SECRET_KEY }}
  RING_S3C_ENDPOINT: ${{ secrets.RING_S3C_BACKEND_ENDPOINT }}
  RING_S3C_BACKEND_SOURCE_LOCATION: rings3cbackendingestion
  RING_S3C_INGESTION_SRC_BUCKET_NAME: ingestion-test-src-bucket-${{ github.run_id }}-${{ github.run_attempt }}
  # CTST end2end tests
  NOTIF_DEST_NAME: "destination1"
  NOTIF_DEST_TOPIC: "destination-topic-1"
  NOTIF_ALT_DEST_NAME: "destination2"
  NOTIF_ALT_DEST_TOPIC: "destination-topic-2"
  SUBDOMAIN: "zenko.local"
  DR_SUBDOMAIN: "dr.zenko.local"
  SKOPEO_PATH: "/tmp"
  HELM_VERSION: "v3.15.4"
  YQ_VERSION: "v4.44.3"
  YQ_BINARY: "yq_linux_amd64"
  KUSTOMIZE_VERSION: "v5.4.3"
  GO_VERSION: "1.23.0"
  SKOPEO_VERSION: "v1.16.1"
  KUBECTL_VERSION: "1.31.0"
  TILT_VERSION: "0.33.19"
  KIND_VERSION: "v0.12.0"
  ZENKO_ENABLE_SOSAPI: false
  EXPIRE_ONE_DAY_EARLIER: true
  TRANSITION_ONE_DAY_EARLIER: true
  TIME_PROGRESSION_FACTOR: 1
  JUNIT_REPORT_PATH: /reports/ctst-junit.xml
  # DEBUG WAIT
  TMATE_SERVER_HOST: ${{ secrets.TMATE_SERVER_HOST }}
  TMATE_SERVER_PORT: ${{ secrets.TMATE_SERVER_PORT }}
  TMATE_SERVER_RSA_FINGERPRINT: ${{ secrets.TMATE_SERVER_RSA_FINGERPRINT }}
  TMATE_SERVER_ED25519_FINGERPRINT: ${{ secrets.TMATE_SERVER_ED25519_FINGERPRINT }}
  # Mocha reporter configuration
  MOCHA_FILE: /reports/test-results-[hash].xml

jobs:
  check-dashboard-versions:
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: install jq
        run: |
          sudo apt-get update && sudo apt-get install -y \
          jq
      - name: Verify monitoring dashboard versions
        run: bash ./.github/scripts/check_versions.sh

  check-workflows:
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup node
        uses: actions/setup-node@v4
        with:
          node-version: '16'
          cache: yarn
          cache-dependency-path: tests/workflows/yarn.lock
      - name: Install dependencies
        run: yarn --cwd tests/workflows install
      - name: Run workflows tests
        run: ACT_LOG=true yarn --cwd tests/workflows test
      - name: Collect logs
        run: |
          mkdir -p /tmp/artifacts/data/${{ github.job }}/act-logs/
          mv tests/workflows/act-*.log /tmp/artifacts/data/check-workflows/act-logs/
          tar zcvf /tmp/artifacts/${{ github.sha }}-${{ github.job }}-act-logs.tgz /tmp/artifacts/data/${{ github.job }}/act-logs;
      - name: Upload artifacts
        uses: scality/action-artifacts@v4
        with:
          method: upload
          url: https://artifacts.scality.net
          user: ${{ secrets.ARTIFACTS_USER }}
          password: ${{ secrets.ARTIFACTS_PASSWORD }}
          source: /tmp/artifacts
        if: always()

  build-doc:
    runs-on: ubuntu-20.04
    env:
      ARTIFACTS_USER: "${{ secrets.ARTIFACTS_USER }}"
      ARTIFACTS_PASSWORD: "${{ secrets.ARTIFACTS_PASSWORD }}"
      DEBIAN_FRONTEND: "noninteractive"
      ROOT_DOCUMENTATION_BUILDDIR: "docs/docsource/build"
      DOCUMENTATION_BUILDDIR: "build"
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install required dependencies
        uses: ./.github/actions/install-doc-dependencies
      - name: Create build directory
        run: mkdir ./${{ env.DOCUMENTATION_BUILDDIR }}
        working-directory: ./docs
      - name: Build docs
        run: tox --workdir /tmp/tox -e docs -- html BUILDDIR=${{ env.DOCUMENTATION_BUILDDIR }}
        working-directory: ./docs
      - name: Create tar file for documentation
        run: |
          tar --create --gzip --directory ./${{ env.ROOT_DOCUMENTATION_BUILDDIR }} --file ./${{ env.ROOT_DOCUMENTATION_BUILDDIR }}/zenko-documentation.tar.gz doctrees html
      - name: Upload documentation
        uses: scality/action-artifacts@v4
        with:
          method: upload
          url: https://artifacts.scality.net
          user: ${{ secrets.ARTIFACTS_USER }}
          password: ${{ secrets.ARTIFACTS_PASSWORD }}
          source: ${{ env.ROOT_DOCUMENTATION_BUILDDIR }}

  build-iso:
    runs-on: ubuntu-22.04
    needs: [build-kafka, check-dashboard-versions]
    steps:
      - name: Install dependencies
        shell: bash
        run: |-
          sudo curl --fail -LO https://dl.google.com/go/go${{ env.GO_VERSION }}.linux-amd64.tar.gz
          sudo tar -C /usr/local -xzf go${{ env.GO_VERSION }}.linux-amd64.tar.gz
          sudo curl --fail -sSL https://get.helm.sh/helm-${{ env.HELM_VERSION }}-linux-amd64.tar.gz | tar -xvz
          sudo install linux-amd64/helm /usr/local/bin
          sudo curl --fail -L https://github.com/mikefarah/yq/releases/download/${{ env.YQ_VERSION }}/${{ env.YQ_BINARY }} -o /usr/bin/yq
          sudo chmod +x /usr/bin/yq
          sudo curl -sSL https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F${{ env.KUSTOMIZE_VERSION }}/kustomize_${{ env.KUSTOMIZE_VERSION }}_linux_amd64.tar.gz | tar xzvf -
          sudo mv kustomize /usr/local/bin
          sudo apt-get update
          sudo apt-get install libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config hardlink mkisofs isomd5sum
          sudo git clone --depth 1 --branch ${{ env.SKOPEO_VERSION }} https://github.com/containers/skopeo ${{ env.SKOPEO_PATH }}/src/github.com/containers/skopeo
          cd ${{ env.SKOPEO_PATH }}/src/github.com/containers/skopeo && \
            sudo PATH="/usr/local/go/bin:$PATH" DISABLE_DOCS=1 make bin/skopeo && \
            sudo PATH="/usr/local/go/bin:$PATH" DISABLE_DOCS=1 make install
      - name: Checkout
        uses: actions/checkout@v4
      - name: Login to Registry
        uses: docker/login-action@v3
        with:
          username: "${{ github.repository_owner }}"
          password: "${{ github.token }}"
          registry: ghcr.io
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: "${{ secrets.DOCKERHUB_LOGIN }}"
          password: "${{ secrets.DOCKERHUB_PASSWORD }}"
      - name: Configure GIT
        run: git config --global url."https://${GIT_ACCESS_TOKEN}@github.com/".insteadOf "https://github.com/"
        env:
          GIT_ACCESS_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }}
      - name: Build ISO
        run: PATH="/usr/local/bin:$PATH" bash -x ./build.sh
        working-directory: ./solution
      - name: Build base ISO
        run: PATH="/usr/local/bin:$PATH" bash -x ./build.sh
        working-directory: ./solution-base
      - name: Upload ISO
        uses: scality/action-artifacts@v4
        with:
          method: upload
          url: https://artifacts.scality.net
          user: ${{ secrets.ARTIFACTS_USER }}
          password: ${{ secrets.ARTIFACTS_PASSWORD }}
          source: "solution/_build"
      - name: Upload base ISO
        uses: scality/action-artifacts@v4
        with:
          method: upload
          url: https://artifacts.scality.net
          user: ${{ secrets.ARTIFACTS_USER }}
          password: ${{ secrets.ARTIFACTS_PASSWORD }}
          source: "solution-base/_build"

  build-kafka:
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to Registry
        uses: docker/login-action@v3
        with:
          username: "${{ github.repository_owner }}"
          password: "${{ github.token }}"
          registry: ghcr.io

      - name: Extract environment
        run: |-
          solution/kafka_build_vars.sh >> $GITHUB_ENV

      - name: Check kafka & kafka-connect versions match
        run: |-
          [ "${{ env.KAFKA_TAG }}-${{ env.MONGODB_CONNECTOR_TAG }}" = "${{ env.KAFKA_CONNECT_TAG }}" ]

      - name: Build and push kafka
        uses: docker/build-push-action@v5
        with:
          push: true
          context: ./solution/kafka
          build-args: |-
            scala_version=${{ env.scala_version }}
            kafka_version=${{ env.kafka_version }}
          tags: "${{ env.KAFKA_IMAGE }}:${{ env.KAFKA_TAG }}-${{ env.BUILD_TREE_HASH }}"
          cache-from: type=gha,scope=kafka-${{ env.KAFKA_TAG }}
          cache-to: type=gha,mode=max,scope=kafka-${{ env.KAFKA_TAG }}

      - name: Build and push kafka-connect
        uses: docker/build-push-action@v5
        with:
          push: true
          context: ./solution/kafka-connect
          build-args: |-
            JMX_JAVAAGENT_IMAGE=${{ env.JMX_JAVAAGENT_IMAGE }}
            JMX_JAVAAGENT_TAG=${{ env.JMX_JAVAAGENT_TAG }}
            KAFKA_IMAGE=${{ env.KAFKA_IMAGE }}
            KAFKA_TAG=${{ env.KAFKA_TAG }}-${{ env.BUILD_TREE_HASH }}
            MONGODB_CONNECTOR_TAG=${{ env.MONGODB_CONNECTOR_TAG }}
          tags: "${{ env.KAFKA_CONNECT_IMAGE }}:${{ env.KAFKA_CONNECT_TAG }}-${{ env.BUILD_TREE_HASH }}"
          cache-from: type=gha,scope=kafka-connect-${{ env.KAFKA_CONNECT_TAG }}
          cache-to: type=gha,mode=max,scope=kafka-connect-${{ env.KAFKA_CONNECT_TAG }}

  build-test-image:
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to Registry
        uses: docker/login-action@v3
        with:
          username: "${{ github.repository_owner }}"
          password: "${{ github.token }}"
          registry: ghcr.io
      - name: Generate end2end config yaml
        run: |-
          cd tests/zenko_tests
          envsubst < 'e2e-config.yaml.template' > 'e2e-config.yaml'
          if [[ "${ENABLE_RING_TESTS}" == "false" ]]; then
            yq -i 'del(.locations[] | select(.locationType == "location-scality-ring-s3-v1"))' e2e-config.yaml
          fi
          cat e2e-config.yaml
          echo 'Generated e2e-config.yaml file'
      - name: Build and push CI image
        uses: docker/build-push-action@v5
        with:
          push: true
          context: ./tests/zenko_tests
          tags: "${{ env.E2E_IMAGE_NAME }}:${{ env.E2E_IMAGE_TAG }}"
          cache-from: type=gha,scope=end2end-test
          cache-to: type=gha,mode=max,scope=end2end-test

  lint-and-build-ctst:
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          token: ${{ secrets.GIT_ACCESS_TOKEN }}
      - name: Configure GIT
        run: |
          git config --global url.https://x-access-token:${{ secrets.GIT_ACCESS_TOKEN }}@github.com/.insteadOf https://github.com/
          git config --global url.https://x-access-token:${{ secrets.GIT_ACCESS_TOKEN }}@github.com/.insteadOf git@github.com:
          git config --global url.https://x-access-token:${{ secrets.GIT_ACCESS_TOKEN }}@github.com/.insteadOf ssh://git@github.com/
        env:
          GIT_ACCESS_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }}
      - uses: actions/setup-node@v4
        with:
          node-version: '20'
          cache: yarn
          cache-dependency-path: tests/ctst/yarn.lock
      - name: Install ctst test dependencies
        working-directory: tests/ctst
        run: yarn install
      - name: Lint ctst tests
        working-directory: tests/ctst
        run: yarn lint
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to Registry
        uses: docker/login-action@v3
        with:
          username: "${{ github.repository_owner }}"
          password: "${{ github.token }}"
          registry: ghcr.io
      - name: Get CTST image tag
        shell: bash
        run: |-
          echo "CTST_TAG=$(sed 's/.*"cli-testing": ".*#\(.*\)".*/\1/;t;d' ./tests/ctst/package.json)" >> $GITHUB_ENV
      - name: Extract environment
        working-directory: solution
        run: |-
          cat <<EOF >> $GITHUB_ENV
          SORBET_TAG=$(yq eval '.sorbet.tag' deps.yaml)
          DRCTL_TAG=$(yq eval .drctl.tag deps.yaml)
          EOF
      - name: Build and push CI image
        uses: docker/build-push-action@v5
        with:
          push: true
          context: ./tests/ctst
          build-args: |
            CTST_TAG=${{ env.CTST_TAG }}
            SORBET_TAG=${{ env.SORBET_TAG }}
            DRCTL_TAG=${{ env.DRCTL_TAG}}
          tags: "${{ env.E2E_CTST_IMAGE_NAME }}:${{ env.E2E_IMAGE_TAG }}"
          cache-from: type=gha,scope=end2end-ctst
          cache-to: type=gha,mode=max,scope=end2end-ctst

  end2end-http:
    needs: [build-kafka, build-test-image, check-dashboard-versions]
    runs-on:
      - ubuntu
      - focal
      - xlarge
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install dependencies
        uses: ./.github/actions/install-end2end-dependencies
      - name: Wait for Docker daemon to be ready
        uses: ./.github/actions/wait-docker-ready
      - name: Login to Registry
        uses: docker/login-action@v3
        with:
          username: "${{ github.repository_owner }}"
          password: "${{ github.token }}"
          registry: ghcr.io
      - name: Deploy Zenko
        uses: ./.github/actions/deploy
        env:
          GIT_ACCESS_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }}
      - name: Run init CI test
        run: bash run-e2e-test.sh "end2end" ${E2E_IMAGE_NAME}:${E2E_IMAGE_TAG} "end2end" "default"
        working-directory: ./.github/scripts/end2end
      - name: Run iam policies tests
        run: bash run-e2e-test.sh "end2end" ${E2E_IMAGE_NAME}:${E2E_IMAGE_TAG} "iam-policies" "default"
        working-directory: ./.github/scripts/end2end
      - name: Run cloudserver tests
        run: bash run-e2e-test.sh "end2end" ${E2E_IMAGE_NAME}:${E2E_IMAGE_TAG} "object-api" "default"
        working-directory: ./.github/scripts/end2end
      - name: Run smoke tests
        run: bash run-e2e-test.sh "end2end" ${E2E_IMAGE_NAME}:${E2E_IMAGE_TAG} "smoke" "default"
        working-directory: ./.github/scripts/end2end
      - name: Debug wait
        uses: ./.github/actions/debug-wait
        timeout-minutes: 60
        if: failure() && runner.debug == '1'
      - name: Upload results
        if: "!cancelled() && env.TRUNK_TOKEN"
        uses: trunk-io/analytics-uploader@main
        with:
          junit-paths: /artifacts/data/reports/*.xml
          org-slug: ${{ github.repository_owner }}
          token: ${{ env.TRUNK_TOKEN }}
        env:
          TRUNK_TOKEN: ${{ secrets.TRUNK_TOKEN }}
        continue-on-error: true
      - name: Archive artifact logs and data
        uses: ./.github/actions/archive-artifacts
        env:
          STAGE: ${{ github.job }}.${{ github.run_attempt }}
        if: always()
      - name: Clean Up
        run: kind delete cluster
      - name: Upload artifacts
        uses: scality/action-artifacts@v4
        with:
          method: upload
          url: https://artifacts.scality.net
          user: ${{ secrets.ARTIFACTS_USER }}
          password: ${{ secrets.ARTIFACTS_PASSWORD }}
          source: /tmp/artifacts
        if: always()

  end2end-pra:
    needs: [build-kafka, check-dashboard-versions, lint-and-build-ctst]
    runs-on: ubuntu-22.04-16core
    env:
      GIT_ACCESS_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install dependencies
        uses: ./.github/actions/install-end2end-dependencies
      - name: Wait for Docker daemon to be ready
        uses: ./.github/actions/wait-docker-ready
      - name: Login to Registry
        uses: docker/login-action@v3
        with:
          username: "${{ github.repository_owner }}"
          password: "${{ github.token }}"
          registry: ghcr.io
      - name: Deploy Zenko
        uses: ./.github/actions/deploy
      - name: Prepare PRA environment
        run: bash prepare-pra.sh
        working-directory: ./.github/scripts/end2end
      - name: Deploy second Zenko for PRA
        run: bash deploy-zenko.sh end2end-pra default './configs/zenko.yaml'
        env:
          ZENKO_MONGODB_DATABASE: "pradb"
        working-directory: ./.github/scripts/end2end
      - name: Add Keycloak pra user and assign StorageManager role
        shell: bash
        run: bash keycloak-helper.sh add-user default end2end-pra
        env:
          OIDC_USERNAME: 'zenko-end2end-pra'
          OIDC_EMAIL: 'e2e-pra@zenko.local'
        working-directory: ./.github/scripts/end2end
      - name: Configure E2E PRA test environment
        run: bash configure-e2e.sh end2end-pra ${E2E_IMAGE_NAME}:${E2E_IMAGE_TAG} default
        working-directory: ./.github/scripts/end2end
        env:
          OIDC_USERNAME: 'zenko-end2end-pra'
      - name: Configure E2E CTST test environment
        run: bash configure-e2e-ctst.sh
        working-directory: ./.github/scripts/end2end
      - name: Run CTST end to end tests
        run: bash run-e2e-ctst.sh "" "" "" "" --tags @PRA
        working-directory: ./.github/scripts/end2end
      - name: Debug wait
        uses: ./.github/actions/debug-wait
        timeout-minutes: 60
        if: failure() && runner.debug == '1'
      - name: Upload results
        if: env.TRUNK_TOKEN && !cancelled()
        uses: trunk-io/analytics-uploader@main
        with:
          junit-paths: /artifacts/data/reports/*.xml
          org-slug: ${{ github.repository_owner }}
          token: ${{ env.TRUNK_TOKEN }}
        env:
          TRUNK_TOKEN: ${{ secrets.TRUNK_TOKEN }}
        continue-on-error: true
      - name: Archive artifact logs and data
        uses: ./.github/actions/archive-artifacts
        env:
          STAGE: ${{ github.job }}.${{ github.run_attempt }}
        if: always()
      - name: Clean Up
        run: kind delete cluster
      - name: Upload artifacts
        uses: scality/action-artifacts@v4
        with:
          method: upload
          url: https://artifacts.scality.net
          user: ${{ secrets.ARTIFACTS_USER }}
          password: ${{ secrets.ARTIFACTS_PASSWORD }}
          source: /tmp/artifacts
        if: always()

  end2end-https:
    needs: [build-kafka, build-test-image, check-dashboard-versions]
    runs-on:
      - ubuntu
      - focal
      - xlarge
    env:
      UI_ENDPOINT: 'https://ui.zenko.local'
      OIDC_ENDPOINT: 'https://keycloak.zenko.local'
      NAVBAR_ENDPOINT: 'https://shell-ui.zenko.local'
      ENABLE_KEYCLOAK_HTTPS: 'true'
      GIT_ACCESS_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install dependencies
        uses: ./.github/actions/install-end2end-dependencies
      - name: Wait for Docker daemon to be ready
        uses: ./.github/actions/wait-docker-ready
      - name: Login to Registry
        uses: docker/login-action@v3
        with:
          username: "${{ github.repository_owner }}"
          password: "${{ github.token }}"
          registry: ghcr.io
      - name: Deploy Zenko
        uses: ./.github/actions/deploy
      - name: Run init CI test
        run: bash run-e2e-test.sh "end2end" ${E2E_IMAGE_NAME}:${E2E_IMAGE_TAG} "end2end" "default"
        working-directory: ./.github/scripts/end2end
      - name: Run smoke tests
        run: bash run-e2e-test.sh "end2end" ${E2E_IMAGE_NAME}:${E2E_IMAGE_TAG} "smoke" "default"
        working-directory: ./.github/scripts/end2end
      - name: Debug wait
        uses: ./.github/actions/debug-wait
        timeout-minutes: 60
        if: failure() && runner.debug == '1'
      - name: Upload results
        if: env.TRUNK_TOKEN && !cancelled()
        uses: trunk-io/analytics-uploader@main
        with:
          junit-paths: /artifacts/data/reports/*.xml
          org-slug: ${{ github.repository_owner }}
          token: ${{ env.TRUNK_TOKEN }}
        env:
          TRUNK_TOKEN: ${{ secrets.TRUNK_TOKEN }}
        continue-on-error: true
      # Temporarily disabled as CTST will test the same APIs more
      # extensively.
      # - name: Run vault e2e tests
      #   run: bash vault-e2e-test.sh
      #   working-directory: ./.github/scripts/end2end
      - name: Archive artifact logs and data
        uses: ./.github/actions/archive-artifacts
        env:
          STAGE: ${{ github.job }}.${{ github.run_attempt }}
        if: always()
      - name: Clean Up
        run: kind delete cluster
      - name: Upload artifacts
        uses: scality/action-artifacts@v4
        with:
          method: upload
          url: https://artifacts.scality.net
          user: ${{ secrets.ARTIFACTS_USER }}
          password: ${{ secrets.ARTIFACTS_PASSWORD }}
          source: /tmp/artifacts
        if: always()

  end2end-sharded:
    needs: [build-kafka, build-test-image, check-dashboard-versions]
    runs-on:
      - ubuntu-22.04-8core
      # Enable this for Ring-based tests
      # - scality-cloud
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install dependencies
        uses: ./.github/actions/install-end2end-dependencies
      - name: Wait for Docker daemon to be ready
        uses: ./.github/actions/wait-docker-ready
      - name: Login to Registry
        uses: docker/login-action@v3
        with:
          username: "${{ github.repository_owner }}"
          password: "${{ github.token }}"
          registry: ghcr.io
      - name: Deploy Zenko
        uses: ./.github/actions/deploy
        env:
          GIT_ACCESS_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }}
      - name: Run backbeat end to end tests
        run: bash run-e2e-test.sh "end2end" ${E2E_IMAGE_NAME}:${E2E_IMAGE_TAG} "backbeat" "default"
        working-directory: ./.github/scripts/end2end
      - name: Upload results
        if: env.TRUNK_TOKEN && !cancelled()
        uses: trunk-io/analytics-uploader@main
        with:
          junit-paths: /artifacts/data/reports/*.xml
          org-slug: ${{ github.repository_owner }}
          token: ${{ env.TRUNK_TOKEN }}
        env:
          TRUNK_TOKEN: ${{ secrets.TRUNK_TOKEN }}
        continue-on-error: true
      - name: Debug wait
        uses: ./.github/actions/debug-wait
        timeout-minutes: 60
        if: failure() && runner.debug == '1'
      - name: Archive artifact logs and data
        uses: ./.github/actions/archive-artifacts
        env:
          STAGE: ${{ github.job }}.${{ github.run_attempt }}
        if: always()
      - name: Clean Up
        run: kind delete cluster
      - name: Upload artifacts
        uses: scality/action-artifacts@v4
        with:
          method: upload
          url: https://artifacts.scality.net
          user: ${{ secrets.ARTIFACTS_USER }}
          password: ${{ secrets.ARTIFACTS_PASSWORD }}
          source: /tmp/artifacts
        if: always()

  ctst-end2end-sharded:
    needs: [build-kafka, lint-and-build-ctst, check-dashboard-versions]
    runs-on:
      - ubuntu-22.04-8core
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install dependencies
        uses: ./.github/actions/install-end2end-dependencies
      - name: Wait for Docker daemon to be ready
        uses: ./.github/actions/wait-docker-ready
      - name: Kubectl tool installer
        uses: Azure/setup-kubectl@v4
      - name: Login to Registry
        uses: docker/login-action@v3
        with:
          username: "${{ github.repository_owner }}"
          password: "${{ github.token }}"
          registry: ghcr.io
      - name: Deploy Zenko
        uses: ./.github/actions/deploy
        env:
          GIT_ACCESS_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }}
          ZENKO_ENABLE_SOSAPI: true
          TIME_PROGRESSION_FACTOR: 86400
          TRANSITION_ONE_DAY_EARLIER: false
          EXPIRE_ONE_DAY_EARLIER: false
      - name: Configure E2E test environment
        run: bash configure-e2e-ctst.sh
        working-directory: ./.github/scripts/end2end
      - name: Run CTST end to end tests
        run: bash run-e2e-ctst.sh "" "" "" "" --tags 'not @PRA'
        working-directory: ./.github/scripts/end2end
      - name: Upload results
        if: env.TRUNK_TOKEN && !cancelled()
        uses: trunk-io/analytics-uploader@main
        with:
          junit-paths: /artifacts/data/reports/*.xml
          org-slug: ${{ github.repository_owner }}
          token: ${{ env.TRUNK_TOKEN }}
        env:
          TRUNK_TOKEN: ${{ secrets.TRUNK_TOKEN }}
        continue-on-error: true
      - name: Debug wait
        uses: ./.github/actions/debug-wait
        timeout-minutes: 60
        if: failure() && runner.debug == '1'
      - name: Archive artifact logs and data
        uses: ./.github/actions/archive-artifacts
        env:
          STAGE: ${{ github.job }}.${{ github.run_attempt }}
        if: always()
      - name: Clean Up
        run: kind delete cluster
      - name: Upload artifacts
        uses: scality/action-artifacts@v4
        with:
          method: upload
          url: https://artifacts.scality.net
          user: ${{ secrets.ARTIFACTS_USER }}
          password: ${{ secrets.ARTIFACTS_PASSWORD }}
          source: /tmp/artifacts
        if: always()

  write-final-status:
    runs-on: ubuntu-latest
    needs:
      - check-dashboard-versions
      - check-workflows
      - build-doc
      - build-iso
      - build-kafka
      - build-test-image
      - end2end-http
      - end2end-https
      - end2end-sharded
      - end2end-pra
      - ctst-end2end-sharded
    steps:
      - name: Upload final status
        uses: scality/actions/upload_final_status@1.8.0
        with:
          ARTIFACTS_USER: ${{ secrets.ARTIFACTS_USER }}
          ARTIFACTS_PASSWORD: ${{ secrets.ARTIFACTS_PASSWORD }}
          JOBS_RESULTS: ${{ join(needs.*.result) }}
        if: always()