diff --git a/.azure-pipelines/stage/macos.yml b/.azure-pipelines/stage/macos.yml
index 4b7f99e718d31..fc990eafd737f 100644
--- a/.azure-pipelines/stage/macos.yml
+++ b/.azure-pipelines/stage/macos.yml
@@ -24,7 +24,12 @@ jobs:
   - script: ./ci/mac_ci_setup.sh
     displayName: "Install dependencies"
 
-  - script: ./ci/mac_ci_steps.sh
+  - bash: |
+      set -e
+      GCP_SERVICE_ACCOUNT_KEY_PATH=$(mktemp -t gcp_service_account.XXXXXX.json)
+      bash -c 'echo "$(GcpServiceAccountKey)"' | base64 --decode > "${GCP_SERVICE_ACCOUNT_KEY_PATH}"
+      BAZEL_BUILD_EXTRA_OPTIONS+=" --google_credentials=${GCP_SERVICE_ACCOUNT_KEY_PATH}"
+      ./ci/mac_ci_steps.sh
     displayName: "Run Mac CI"
     env:
       BAZEL_BUILD_EXTRA_OPTIONS: >-
@@ -32,7 +37,6 @@ jobs:
         --flaky_test_attempts=2
         --remote_cache=grpcs://remotebuildexecution.googleapis.com
         --remote_instance_name=projects/envoy-ci/instances/default_instance
-      GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }}
       ENVOY_RBE: 1
 
   - task: PublishTestResults@2
diff --git a/.azure-pipelines/stage/windows.yml b/.azure-pipelines/stage/windows.yml
index 9be16bebcf5c6..fa2729b822545 100644
--- a/.azure-pipelines/stage/windows.yml
+++ b/.azure-pipelines/stage/windows.yml
@@ -26,14 +26,27 @@ jobs:
       key: '"windows.release" | $(cacheKeyBazel)'
       path: $(Build.StagingDirectory)/repository_cache
     continueOnError: true
-  - bash: ci/run_envoy_docker.sh ci/windows_ci_steps.sh
+
+  - bash: |
+      set -e
+      ENVOY_SHARED_TMP_DIR="C:\\Users\\VSSADM~1\\AppData\\Local\\Temp\\bazel-shared"
+      mkdir -p "$ENVOY_SHARED_TMP_DIR"
+      GCP_SERVICE_ACCOUNT_KEY_PATH=$(mktemp -p "${ENVOY_SHARED_TMP_DIR}" -t gcp_service_account.XXXXXX.json)
+      bash -c 'echo "$(GcpServiceAccountKey)"' | base64 --decode > "${GCP_SERVICE_ACCOUNT_KEY_PATH}"
+      export BAZEL_BUILD_EXTRA_OPTIONS+=" --google_credentials=${GCP_SERVICE_ACCOUNT_KEY_PATH}"
+      export ENVOY_SHARED_TMP_DIR
+      ci/run_envoy_docker.sh ci/windows_ci_steps.sh
     displayName: "Run Windows msvc-cl CI"
     env:
       CI_TARGET: "windows"
       ENVOY_DOCKER_BUILD_DIR: "$(Build.StagingDirectory)"
       ENVOY_RBE: "true"
-      BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=rbe-google --config=remote-msvc-cl --jobs=$(RbeJobs) --flaky_test_attempts=2"
-      GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }}
+      BAZEL_BUILD_EXTRA_OPTIONS: >-
+        --config=remote-ci
+        --config=rbe-google
+        --config=remote-msvc-cl
+        --jobs=$(RbeJobs)
+        --flaky_test_attempts=2
 
   - task: PublishTestResults@2
     inputs:
diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh
index d2a54b474bb17..b50bb1190d661 100755
--- a/.devcontainer/setup.sh
+++ b/.devcontainer/setup.sh
@@ -1,10 +1,8 @@
 #!/usr/bin/env bash
 
-. ci/setup_cache.sh
-trap - EXIT # Don't remove the key file written into a temporary file
-
 BAZELRC_FILE=~/.bazelrc bazel/setup_clang.sh /opt/llvm
 
+# TODO(phlax): use user.bazelrc
 # Use generated toolchain config because we know the base container is the one we're using in RBE.
 # Not using libc++ here because clangd will raise some tidy issue in libc++ header as of version 9.
 echo "build --config=rbe-toolchain-clang" >> ~/.bazelrc
diff --git a/ci/do_ci.sh b/ci/do_ci.sh
index c3d8f5f468ac6..8263bde4d9bfa 100755
--- a/ci/do_ci.sh
+++ b/ci/do_ci.sh
@@ -8,8 +8,6 @@ set -e
 export SRCDIR="${SRCDIR:-$PWD}"
 export ENVOY_SRCDIR="${ENVOY_SRCDIR:-$PWD}"
 
-# shellcheck source=ci/setup_cache.sh
-. "$(dirname "$0")"/setup_cache.sh
 # shellcheck source=ci/build_setup.sh
 . "$(dirname "$0")"/build_setup.sh
 
diff --git a/ci/mac_ci_steps.sh b/ci/mac_ci_steps.sh
index 2ab857c72970a..dc779a665c713 100755
--- a/ci/mac_ci_steps.sh
+++ b/ci/mac_ci_steps.sh
@@ -11,9 +11,6 @@ trap finish EXIT
 echo "disk space at beginning of build:"
 df -h
 
-# shellcheck source=ci/setup_cache.sh
-. "$(dirname "$0")"/setup_cache.sh
-
 read -ra BAZEL_BUILD_EXTRA_OPTIONS <<< "${BAZEL_BUILD_EXTRA_OPTIONS:-}"
 read -ra BAZEL_EXTRA_TEST_OPTIONS <<< "${BAZEL_EXTRA_TEST_OPTIONS:-}"
 
diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh
index 2aa752e5baadd..558bfcc229cab 100755
--- a/ci/run_envoy_docker.sh
+++ b/ci/run_envoy_docker.sh
@@ -91,7 +91,7 @@ VOLUMES=(
     -v "${ENVOY_DOCKER_BUILD_DIR}":"${BUILD_DIR_MOUNT_DEST}"
     -v "${SOURCE_DIR}":"${SOURCE_DIR_MOUNT_DEST}")
 
-if ! is_windows && [[ -n "$ENVOY_DOCKER_IN_DOCKER" || -n "$ENVOY_SHARED_TMP_DIR" ]]; then
+if [[ -n "$ENVOY_DOCKER_IN_DOCKER" || -n "$ENVOY_SHARED_TMP_DIR" ]]; then
     # Create a "shared" directory that has the same path in/outside the container
     # This allows the host docker engine to see artefacts using a temporary path created inside the container,
     # at the same path.
diff --git a/ci/setup_cache.sh b/ci/setup_cache.sh
deleted file mode 100755
index ca910ec1a090c..0000000000000
--- a/ci/setup_cache.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/bash
-
-set -e
-
-if [[ -n "${GCP_SERVICE_ACCOUNT_KEY:0:1}" ]]; then
-  # mktemp will create a tempfile with u+rw permission minus umask, it will not be readable by all
-  # users by default.
-  GCP_SERVICE_ACCOUNT_KEY_FILE=$(mktemp -t gcp_service_account.XXXXXX.json)
-
-  gcp_service_account_cleanup() {
-    echo "Deleting service account key file..."
-    rm -rf "${GCP_SERVICE_ACCOUNT_KEY_FILE}"
-  }
-
-  trap gcp_service_account_cleanup EXIT
-
-  echo "Setting GCP_SERVICE_ACCOUNT_KEY is deprecated, please place your decoded GCP key in " \
-       "an exported/shared tmp directory and add it to BAZEL_BUILD_EXTRA_OPTIONS, eg: " >&2
-  # shellcheck disable=SC2086
-  echo "$ export ENVOY_SHARED_TMP_DIR=/tmp/envoy-shared" \
-       "$ ENVOY_RBE_KEY_PATH=$(mktemp -p \"${ENVOY_SHARED_TMP_DIR}\" -t gcp_service_account.XXXXXX.json)" \
-       "$ bash -c 'echo \"$(GcpServiceAccountKey)\"' | base64 --decode > \"${ENVOY_RBE_KEY_PATH}\"" \
-       "$ export BAZEL_BUILD_EXTRA_OPTIONS+=\" --google_credentials=${ENVOY_RBE_KEY_PATH}\"" >&2
-  bash -c 'echo "${GCP_SERVICE_ACCOUNT_KEY}"' | base64 --decode > "${GCP_SERVICE_ACCOUNT_KEY_FILE}"
-  export BAZEL_BUILD_EXTRA_OPTIONS+=" --google_credentials=${GCP_SERVICE_ACCOUNT_KEY_FILE}"
-fi
-
-if [[ -n "${BAZEL_REMOTE_CACHE}" ]]; then
-    echo "Setting BAZEL_REMOTE_CACHE is deprecated, please use BAZEL_BUILD_EXTRA_OPTIONS " \
-         "or use a user.bazelrc config " >&2
-    export BAZEL_BUILD_EXTRA_OPTIONS+=" --remote_cache=${BAZEL_REMOTE_CACHE}"
-    echo "Set up bazel remote read/write cache at ${BAZEL_REMOTE_CACHE}."
-    if [[ -n "${BAZEL_REMOTE_INSTANCE}" ]]; then
-        export BAZEL_BUILD_EXTRA_OPTIONS+=" --remote_instance_name=${BAZEL_REMOTE_INSTANCE}"
-        echo "instance_name: ${BAZEL_REMOTE_INSTANCE}."
-    fi
-fi
diff --git a/ci/windows_ci_steps.sh b/ci/windows_ci_steps.sh
index 58fd0a9a81d58..c16d7392602ac 100755
--- a/ci/windows_ci_steps.sh
+++ b/ci/windows_ci_steps.sh
@@ -11,9 +11,6 @@ trap finish EXIT
 echo "disk space at beginning of build:"
 df -h
 
-# shellcheck source=ci/setup_cache.sh
-. "$(dirname "$0")"/setup_cache.sh
-
 [ -z "${ENVOY_SRCDIR}" ] && export ENVOY_SRCDIR=/c/source
 
 read -ra BAZEL_STARTUP_OPTIONS <<< "${BAZEL_STARTUP_OPTIONS:-}"