Up-port tap/extproc changes from 1.26 to 1.27

.azure-pipelines/ci.yml

@@ -176,31 +176,68 @@ steps:
     tmpfsDockerDisabled: "${{ parameters.tmpfsDockerDisabled }}"
 
 - script: |
-    if [[ "${{ parameters.bazelUseBES }}" == 'false' ]]; then
-        unset GOOGLE_BES_PROJECT_ID
+    ENVOY_SHARED_TMP_DIR=/tmp/bazel-shared
+    mkdir -p "$ENVOY_SHARED_TMP_DIR"
+    BAZEL_BUILD_EXTRA_OPTIONS="${{ parameters.bazelBuildExtraOptions }}"
+    if [[ "${{ parameters.rbe }}" == "True" ]]; then
+        # mktemp will create a tempfile with u+rw permission minus umask, it will not be readable by all
+        # users by default.
+        GCP_SERVICE_ACCOUNT_KEY_PATH=$(mktemp -p "${ENVOY_SHARED_TMP_DIR}" -t gcp_service_account.XXXXXX.json)
+        bash -c 'echo "$(GcpServiceAccountKey)"' | base64 --decode > "${GCP_SERVICE_ACCOUNT_KEY_PATH}"
+        BAZEL_BUILD_EXTRA_OPTIONS+=" ${{ parameters.bazelConfigRBE }} --google_credentials=${GCP_SERVICE_ACCOUNT_KEY_PATH}"
+        ENVOY_RBE=1
+        if [[ "${{ parameters.bazelUseBES }}" == "True" && -n "${GOOGLE_BES_PROJECT_ID}" ]]; then
+            BAZEL_BUILD_EXTRA_OPTIONS+=" --config=rbe-google-bes --bes_instance_name=${GOOGLE_BES_PROJECT_ID}"
+        fi
+    else
+        echo "using local build cache."
+        # Normalize branches - `release/vX.xx`, `vX.xx`, `vX.xx.x` -> `vX.xx`
+        TARGET_BRANCH=$(echo "${CI_TARGET_BRANCH}" | cut -d/ -f2-)
+        BRANCH_NAME="$(echo "${TARGET_BRANCH}" | cut -d/ -f2 | cut -d. -f-2)"
+        if [[ "$BRANCH_NAME" == "merge" ]]; then
+            # Manually run PR commit - there is no easy way of telling which branch
+            # it is, so just set it to `main` - otherwise it tries to cache as `branch/merge`
+            BRANCH_NAME=main
+        fi
+        BAZEL_REMOTE_INSTANCE="branch/${BRANCH_NAME}"
+        echo "instance_name: ${BAZEL_REMOTE_INSTANCE}."
+        BAZEL_BUILD_EXTRA_OPTIONS+=" --config=ci --config=cache-local --remote_instance_name=${BAZEL_REMOTE_INSTANCE} --remote_timeout=600"
     fi
-    ci/run_envoy_docker.sh 'ci/do_ci.sh fetch-${{ parameters.ciTarget }}'
-  condition: and(not(canceled()), not(failed()), ne('${{ parameters.cacheName }}', ''), ne(variables.CACHE_RESTORED, 'true'))
+    if [[ "${{ parameters.cacheTestResults }}" != "True" ]]; then
+        VERSION_DEV="$(cut -d- -f2 "VERSION.txt")"
+        # Use uncached test results for non-release scheduledruns.
+        if [[ $VERSION_DEV == "dev" ]]; then
+            BAZEL_EXTRA_TEST_OPTIONS+=" --nocache_test_results"
+        fi
+    fi
+    # Any PR or CI run in envoy-presubmit uses the fake SCM hash
+    if [[ "${{ variables['Build.Reason'] }}" == "PullRequest" || "${{ variables['Build.DefinitionName'] }}" == 'envoy-presubmit' ]]; then
+        # sha1sum of `ENVOY_PULL_REQUEST`
+        BAZEL_FAKE_SCM_REVISION=e3b4a6e9570da15ac1caffdded17a8bebdc7dfc9
+    fi
+    echo "##vso[task.setvariable variable=BAZEL_BUILD_EXTRA_OPTIONS]${BAZEL_BUILD_EXTRA_OPTIONS}"
+    echo "##vso[task.setvariable variable=BAZEL_EXTRA_TEST_OPTIONS]${BAZEL_EXTRA_TEST_OPTIONS}"
+    echo "##vso[task.setvariable variable=BAZEL_FAKE_SCM_REVISION]${BAZEL_FAKE_SCM_REVISION}"
+    echo "##vso[task.setvariable variable=BAZEL_STARTUP_EXTRA_OPTIONS]${{ parameters.bazelStartupExtraOptions }}"
+    echo "##vso[task.setvariable variable=CI_TARGET_BRANCH]${CI_TARGET_BRANCH}"
+    echo "##vso[task.setvariable variable=ENVOY_BUILD_FILTER_EXAMPLE]${{ parameters.envoyBuildFilterExample }}"
+    echo "##vso[task.setvariable variable=ENVOY_DOCKER_BUILD_DIR]$(Build.StagingDirectory)"
+    echo "##vso[task.setvariable variable=ENVOY_RBE]${ENVOY_RBE}"
+    echo "##vso[task.setvariable variable=ENVOY_SHARED_TMP_DIR]${ENVOY_SHARED_TMP_DIR}"
+    echo "##vso[task.setvariable variable=GCP_SERVICE_ACCOUNT_KEY_PATH]${GCP_SERVICE_ACCOUNT_KEY_PATH}"
+    echo "##vso[task.setvariable variable=GITHUB_TOKEN]${{ parameters.authGithub }}"
   workingDirectory: $(Build.SourcesDirectory)
   env:
-    ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
-    GITHUB_TOKEN: "${{ parameters.authGithub }}"
-    BAZEL_STARTUP_EXTRA_OPTIONS: "${{ parameters.bazelStartupExtraOptions }}"
     ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
       CI_TARGET_BRANCH: "origin/$(System.PullRequest.TargetBranch)"
     ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
       CI_TARGET_BRANCH: "origin/$(Build.SourceBranchName)"
-    # Any PR or CI run in envoy-presubmit uses the fake SCM hash
-    ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.DefinitionName'], 'envoy-presubmit')) }}:
-      # sha1sum of `ENVOY_PULL_REQUEST`
-      BAZEL_FAKE_SCM_REVISION: e3b4a6e9570da15ac1caffdded17a8bebdc7dfc9
-    ${{ if parameters.rbe }}:
-      GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
-      ENVOY_RBE: "1"
-      BAZEL_BUILD_EXTRA_OPTIONS: "${{ parameters.bazelConfigRBE }} ${{ parameters.bazelBuildExtraOptions }}"
-    ${{ if eq(parameters.rbe, false) }}:
-      BAZEL_BUILD_EXTRA_OPTIONS: "--config=ci ${{ parameters.bazelBuildExtraOptions }}"
-      BAZEL_REMOTE_CACHE: $(LocalBuildCache)
+  displayName: "CI env ${{ parameters.ciTarget }}"
+
+- script: ci/run_envoy_docker.sh 'ci/do_ci.sh fetch-${{ parameters.ciTarget }}'
+  condition: and(not(canceled()), not(failed()), ne('${{ parameters.cacheName }}', ''), ne(variables.CACHE_RESTORED, 'true'))
+  workingDirectory: $(Build.SourcesDirectory)
+  env:
     ${{ each var in parameters.env }}:
       ${{ var.key }}: ${{ var.value }}
   displayName: "Fetch assets (${{ parameters.ciTarget }})"
@@ -231,34 +268,10 @@ steps:
   displayName: "Enable IPv6"
   condition: ${{ parameters.managedAgent }}
 
-- script: |
-    if [[ "${{ parameters.bazelUseBES }}" == 'false' ]]; then
-        unset GOOGLE_BES_PROJECT_ID
-    fi
-    ci/run_envoy_docker.sh 'ci/do_ci.sh ${{ parameters.ciTarget }}'
+- script: ci/run_envoy_docker.sh 'ci/do_ci.sh ${{ parameters.ciTarget }}'
   workingDirectory: $(Build.SourcesDirectory)
   env:
-    ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
     ENVOY_BUILD_FILTER_EXAMPLE: ${{ parameters.envoyBuildFilterExample }}
-    GITHUB_TOKEN: "${{ parameters.authGithub }}"
-    BAZEL_STARTUP_EXTRA_OPTIONS: "${{ parameters.bazelStartupExtraOptions }}"
-    ${{ if ne(parameters['cacheTestResults'], true) }}:
-      BAZEL_NO_CACHE_TEST_RESULTS: 1
-    ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
-      CI_TARGET_BRANCH: "origin/$(System.PullRequest.TargetBranch)"
-    ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
-      CI_TARGET_BRANCH: "origin/$(Build.SourceBranchName)"
-    # Any PR or CI run in envoy-presubmit uses the fake SCM hash
-    ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.DefinitionName'], 'envoy-presubmit')) }}:
-      # sha1sum of `ENVOY_PULL_REQUEST`
-      BAZEL_FAKE_SCM_REVISION: e3b4a6e9570da15ac1caffdded17a8bebdc7dfc9
-    ${{ if parameters.rbe }}:
-      GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
-      ENVOY_RBE: "1"
-      BAZEL_BUILD_EXTRA_OPTIONS: "${{ parameters.bazelConfigRBE }} ${{ parameters.bazelBuildExtraOptions }}"
-    ${{ if eq(parameters.rbe, false) }}:
-      BAZEL_BUILD_EXTRA_OPTIONS: "--config=ci ${{ parameters.bazelBuildExtraOptions }}"
-      BAZEL_REMOTE_CACHE: $(LocalBuildCache)
     ${{ each var in parameters.env }}:
       ${{ var.key }}: ${{ var.value }}
   displayName: "Run CI script ${{ parameters.ciTarget }}"
@@ -296,6 +309,13 @@ steps:
   - ${{ each pair in step }}:
       ${{ pair.key }}: ${{ pair.value }}
 
+- bash: |
+    if [[ -n "$GCP_SERVICE_ACCOUNT_KEY_PATH" && -e "$GCP_SERVICE_ACCOUNT_KEY_PATH" ]]; then
+        echo "Removed key: ${GCP_SERVICE_ACCOUNT_KEY_PATH}"
+        rm -rf "$GCP_SERVICE_ACCOUNT_KEY_PATH"
+    fi
+  condition: not(canceled())
+
 - script: |
     set -e
     sudo .azure-pipelines/docker/save_cache.sh "$(Build.StagingDirectory)" /mnt/cache/all true true

.azure-pipelines/env.yml

@@ -148,6 +148,8 @@ jobs:
       RUN_CHECKS=true
       RUN_DOCKER=true
       RUN_PACKAGING=true
+      RUN_RELEASE_TESTS=true
+
       if [[ "$(changed.mobileOnly)" == true || "$(changed.docsOnly)" == true ]]; then
           RUN_BUILD=false
           RUN_DOCKER=false
@@ -156,10 +158,15 @@ jobs:
           RUN_CHECKS=false
           RUN_PACKAGING=false
       fi
+      if [[ "$ISSTABLEBRANCH" == True && -n "$POSTSUBMIT" && "$(state.isDev)" == false ]]; then
+          RUN_RELEASE_TESTS=false
+      fi
+
       echo "##vso[task.setvariable variable=build;isoutput=true]${RUN_BUILD}"
       echo "##vso[task.setvariable variable=checks;isoutput=true]${RUN_CHECKS}"
       echo "##vso[task.setvariable variable=docker;isoutput=true]${RUN_DOCKER}"
       echo "##vso[task.setvariable variable=packaging;isoutput=true]${RUN_PACKAGING}"
+      echo "##vso[task.setvariable variable=releaseTests;isoutput=true]${RUN_RELEASE_TESTS}"
 
     displayName: "Decide what to run"
     workingDirectory: $(Build.SourcesDirectory)
@@ -211,6 +218,7 @@ jobs:
       echo "env.outputs['run.build']: $(run.build)"
       echo "env.outputs['run.checks']: $(run.checks)"
       echo "env.outputs['run.packaging']: $(run.packaging)"
+      echo "env.outputs['run.releaseTests']: $(run.releaseTests)"
       echo
       echo "env.outputs['publish.githubRelease']: $(publish.githubRelease)"
       echo "env.outputs['publish.dockerhub]: $(publish.dockerhub)"

.azure-pipelines/pipelines.yml

@@ -65,6 +65,13 @@ stages:
 # Presubmit/default
 - ${{ if eq(variables.pipelineDefault, true) }}:
   - template: stages.yml
+    parameters:
+      buildStageDeps:
+      - env
+      macBuildStageDeps:
+      - env
+      windowsBuildStageDeps:
+      - env
 
 # Scheduled run anywhere
 - ${{ if eq(variables.pipelineScheduled, true) }}:

.azure-pipelines/stage/checks.yml

@@ -101,15 +101,7 @@ jobs:
         displayName: "Upload $(CI_TARGET) Report to GCS"
         condition: and(not(canceled()), or(eq(variables['CI_TARGET'], 'coverage'), eq(variables['CI_TARGET'], 'fuzz_coverage')))
         env:
-          ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
-          ENVOY_RBE: "1"
-          BAZEL_BUILD_EXTRA_OPTIONS: "--config=ci --config=rbe-google --jobs=$(RbeJobs)"
-          GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }}
           GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }}
-          ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
-            BAZEL_REMOTE_INSTANCE_BRANCH: "$(System.PullRequest.TargetBranch)"
-          ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
-            BAZEL_REMOTE_INSTANCE_BRANCH: "$(Build.SourceBranchName)"
 
 - job: complete
   displayName: "Checks complete"

.azure-pipelines/stage/linux.yml

@@ -11,6 +11,10 @@ parameters:
   displayName: "Artifact suffix"
   type: string
   default:
+- name: runTests
+  displayName: "Run release tests"
+  type: string
+  default: true
 - name: rbe
   displayName: "Use RBE"
   type: boolean
@@ -45,10 +49,19 @@ jobs:
   timeoutInMinutes: ${{ parameters.timeoutBuild }}
   pool: ${{ parameters.pool }}
   steps:
+  - bash: |
+      if [[ "${{ parameters.runTests }}" == "false" ]]; then
+          CI_TARGET="release.server_only"
+      else
+          CI_TARGET="release"
+      fi
+      echo "${CI_TARGET}"
+      echo "##vso[task.setvariable variable=value;isoutput=true]${CI_TARGET}"
+    name: target
   - template: ../ci.yml
     parameters:
       managedAgent: ${{ parameters.managedAgent }}
-      ciTarget: release
+      ciTarget: $(target.value)
       cacheName: "release"
       bazelBuildExtraOptions: ${{ parameters.bazelBuildExtraOptions }}
       cacheTestResults: ${{ parameters.cacheTestResults }}

.azure-pipelines/stage/macos.yml

@@ -24,13 +24,19 @@ jobs:
   - script: ./ci/mac_ci_setup.sh
     displayName: "Install dependencies"
 
-  - script: ./ci/mac_ci_steps.sh
+  - bash: |
+      set -e
+      GCP_SERVICE_ACCOUNT_KEY_PATH=$(mktemp -t gcp_service_account.XXXXXX.json)
+      bash -c 'echo "$(GcpServiceAccountKey)"' | base64 --decode > "${GCP_SERVICE_ACCOUNT_KEY_PATH}"
+      BAZEL_BUILD_EXTRA_OPTIONS+=" --google_credentials=${GCP_SERVICE_ACCOUNT_KEY_PATH}"
+      ./ci/mac_ci_steps.sh
     displayName: "Run Mac CI"
     env:
-      BAZEL_BUILD_EXTRA_OPTIONS: "--remote_download_toplevel --flaky_test_attempts=2"
-      BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
-      BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
-      GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }}
+      BAZEL_BUILD_EXTRA_OPTIONS: >-
+        --remote_download_toplevel
+        --flaky_test_attempts=2
+        --remote_cache=grpcs://remotebuildexecution.googleapis.com
+        --remote_instance_name=projects/envoy-ci/instances/default_instance
       ENVOY_RBE: 1
 
   - task: PublishTestResults@2

.azure-pipelines/stage/prechecks.yml

@@ -25,11 +25,25 @@ parameters:
   type: string
   default: ""
 
+# Timeout/s
+- name: timeoutPrechecks
+  type: number
+  # Building the rst from protos can take a while even with RBE if there is
+  # a lot of change - eg protobuf changed, or a primitve proto changed.
+  default: 40
+
+- name: runPrechecks
+  displayName: "Run prechecks"
+  type: string
+  default: true
 
 jobs:
 - job: prechecks
   displayName: Precheck
-  timeoutInMinutes: 30
+  timeoutInMinutes: ${{ parameters.timeoutPrechecks }}
+  condition: |
+    and(not(canceled()),
+        eq(${{ parameters.runPrechecks }}, 'true'))
   pool:
     vmImage: $(agentUbuntu)
   variables:
@@ -85,15 +99,15 @@ jobs:
           authGPGKey: ${{ parameters.authGPGKey }}
           # GNUPGHOME inside the container
           pathGPGConfiguredHome: /build/.gnupg
-          pathGPGHome: /tmp/envoy-docker-build/.gnupg
+          pathGPGHome: $(Build.StagingDirectory)/.gnupg
       - bash: |
           set -e
           ci/run_envoy_docker.sh "
               echo AUTHORITY > /tmp/authority \
               && gpg --clearsign /tmp/authority \
               && cat /tmp/authority.asc \
               && gpg --verify /tmp/authority.asc"
-          rm -rf /tmp/envoy-docker-build/.gnupg
+          rm -rf $(Build.StagingDirectory)/.gnupg
         displayName: "Ensure container CI can sign with GPG"
         condition: and(not(canceled()), eq(variables['CI_TARGET'], 'docs'))
 
@@ -115,10 +129,6 @@ jobs:
           ci/run_envoy_docker.sh 'ci/do_ci.sh dockerhub-readme'
         displayName: "Dockerhub publishing test"
         env:
-          ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
-          ENVOY_RBE: "1"
-          BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=rbe-google --jobs=$(RbeJobs)"
-          GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }}
           GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }}
         condition: eq(variables['CI_TARGET'], 'docs')
 
@@ -141,14 +151,9 @@ jobs:
         condition: and(failed(), eq(variables['CI_TARGET'], 'check_and_fix_proto_format'))
 
       # Publish docs
-      - script: |
-          ci/run_envoy_docker.sh 'ci/do_ci.sh docs-upload'
+      - script: ci/run_envoy_docker.sh 'ci/do_ci.sh docs-upload'
         displayName: "Upload Docs to GCS"
         env:
-          ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
-          ENVOY_RBE: "1"
-          BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=rbe-google --jobs=$(RbeJobs)"
-          GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }}
           GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }}
         condition: eq(variables['CI_TARGET'], 'docs')
 

.azure-pipelines/stage/publish.yml

@@ -123,10 +123,6 @@ jobs:
               eq(${{ parameters.publishDockerhub }}, 'true'))
         displayName: "Publish Dockerhub description and README"
         env:
-          ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
-          ENVOY_RBE: "1"
-          BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=rbe-google --jobs=$(RbeJobs)"
-          GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }}
           GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }}
           DOCKERHUB_USERNAME: ${{ parameters.authDockerUser }}
           DOCKERHUB_PASSWORD: ${{ parameters.authDockerPassword }}
@@ -277,6 +273,16 @@ jobs:
   pool:
     vmImage: $(agentUbuntu)
   steps:
+  - task: DownloadSecureFile@1
+    name: WorkflowTriggerKey
+    displayName: 'Download workflow trigger key'
+    inputs:
+      secureFile: '${{ parameters.authGithubWorkflow }}'
+  - bash: |
+      set -e
+      KEY="$(cat $(WorkflowTriggerKey.secureFilePath) | base64 -w0)"
+      echo "##vso[task.setvariable variable=value;isoutput=true]$KEY"
+    name: key
   - template: ../ci.yml
     parameters:
       ciTarget: verify.trigger
@@ -310,13 +316,3 @@ jobs:
           mkdir -p $(Build.StagingDirectory)/release.signed
           mv release.signed.tar.zst $(Build.StagingDirectory)/release.signed
         displayName: Fetch signed release
-      - task: DownloadSecureFile@1
-        name: WorkflowTriggerKey
-        displayName: 'Download workflow trigger key'
-        inputs:
-          secureFile: '${{ parameters.authGithubWorkflow }}'
-      - bash: |
-          set -e
-          KEY="$(cat $(WorkflowTriggerKey.secureFilePath) | base64 -w0)"
-          echo "##vso[task.setvariable variable=value;isoutput=true]$KEY"
-        name: key