From 79c2292530cb1772c232ed08b3cc47675c7ed996 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Mon, 12 Oct 2020 21:36:10 -0700 Subject: [PATCH 1/7] ci: use azp for api and go-control-plane sync Signed-off-by: Lizan Zhou --- .azure-pipelines/pipelines.yml | 50 +++++++++++++++++++++++++++++++ ci/api_mirror.sh | 13 ++++---- ci/do_ci.sh | 13 ++++---- ci/go_mirror.sh | 10 +++++-- ci/run_envoy_docker.sh | 1 + tools/api/generate_go_protobuf.py | 14 +++++---- tools/api_boost/api_boost.py | 1 - 7 files changed, 79 insertions(+), 23 deletions(-) diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml index 31fb5e06fae01..0b46ca296a17f 100644 --- a/.azure-pipelines/pipelines.yml +++ b/.azure-pipelines/pipelines.yml @@ -118,6 +118,8 @@ jobs: - job: filter_example displayName: "filter-example sync" dependsOn: [] + pool: + vmImage: "ubuntu-18.04" condition: and(succeeded(), eq(variables['PostSubmit'], true)) steps: - task: InstallSSHKey@0 @@ -133,6 +135,52 @@ jobs: env: AZP_BRANCH: $(Build.SourceBranch) + - job: api + displayName: "data-plane-api sync" + dependsOn: [] + pool: + vmImage: "ubuntu-18.04" + steps: + - task: InstallSSHKey@0 + inputs: + hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" + sshPublicKey: "$(DataPlaneApiPublicKey)" + sshPassphrase: "$(SshDeployKeyPassphrase)" + sshKeySecureFile: "$(DataPlaneApiPrivateKey)" + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true)) + + - bash: ci/api_mirror.sh + displayName: "Sync data-plane-api" + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true)) + + - job: go_control_plane + displayName: "go-control-plane sync" + dependsOn: [] + steps: + - task: InstallSSHKey@0 + inputs: + hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" + sshPublicKey: "$(GoControlPlanePublicKey)" + sshPassphrase: "$(SshDeployKeyPassphrase)" + sshKeySecureFile: "$(GoControlPlanePrivateKey)" + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true)) + + - bash: | + cp -a ~/.ssh $(Build.StagingDirectory)/ + ci/run_envoy_docker.sh 'ci/go_mirror.sh' + displayName: "Sync go-control-plane" + workingDirectory: $(Build.SourcesDirectory) + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + AZP_BRANCH: $(Build.SourceBranch) + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true)) + - job: bazel displayName: "Linux-x64" dependsOn: ["release"] @@ -141,6 +189,8 @@ jobs: strategy: maxParallel: 3 matrix: + api: + CI_TARGET: "bazel.api" gcc: CI_TARGET: "bazel.gcc" clang_tidy: diff --git a/ci/api_mirror.sh b/ci/api_mirror.sh index 077cdd1d3cfe6..09868f72f1d30 100755 --- a/ci/api_mirror.sh +++ b/ci/api_mirror.sh @@ -3,16 +3,15 @@ set -e CHECKOUT_DIR=../data-plane-api +MAIN_BRANCH="refs/heads/azp_api" +API_MAIN_BRANCH="master-ci-test" -if [ -z "$CIRCLE_PULL_REQUEST" ] && [ "$CIRCLE_BRANCH" == "master" ] -then +if [[ "${AZP_BRANCH}" == "${MAIN_BRANCH}" ]]; then echo "Cloning..." - git clone git@github.com:envoyproxy/data-plane-api "$CHECKOUT_DIR" + git clone git@github.com:envoyproxy/data-plane-api "$CHECKOUT_DIR" -b "${API_MAIN_BRANCH}" - git -C "$CHECKOUT_DIR" config user.name "data-plane-api(CircleCI)" + git -C "$CHECKOUT_DIR" config user.name "data-plane-api(Azure Pipelines)" git -C "$CHECKOUT_DIR" config user.email data-plane-api@users.noreply.github.com - git -C "$CHECKOUT_DIR" fetch - git -C "$CHECKOUT_DIR" checkout -B master origin/master # Determine last envoyproxy/envoy SHA in envoyproxy/data-plane-api MIRROR_MSG="Mirrored from https://github.com/envoyproxy/envoy" @@ -40,6 +39,6 @@ then done echo "Pushing..." - git -C "$CHECKOUT_DIR" push origin master + git -C "$CHECKOUT_DIR" push origin "${API_MAIN_BRANCH}" echo "Done" fi diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 4e908fd6aab48..3e9eb939ae845 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -312,19 +312,20 @@ elif [[ "$CI_TARGET" == "bazel.compile_time_options" ]]; then collect_build_profile build exit 0 elif [[ "$CI_TARGET" == "bazel.api" ]]; then + # Use libstdc++ because the API booster isn't capable of working with libc++ yet. + ENVOY_STDLIB="libstdc++" setup_clang_toolchain + export LLVM_CONFIG="${LLVM_ROOT}"/bin/llvm-config echo "Validating API structure..." ./tools/api/validate_structure.py + echo "Testing API and API Boosting..." + bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild @envoy_api_canonical//test/... @envoy_api_canonical//tools/... \ + @envoy_api_canonical//tools:tap2pcap_test @envoy_dev//clang_tools/api_booster/... echo "Building API..." bazel build "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild @envoy_api_canonical//envoy/... - echo "Testing API..." - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild @envoy_api_canonical//test/... @envoy_api_canonical//tools/... \ - @envoy_api_canonical//tools:tap2pcap_test - echo "Testing API boosting (unit tests)..." - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild @envoy_dev//clang_tools/api_booster/... echo "Testing API boosting (golden C++ tests)..." # We use custom BAZEL_BUILD_OPTIONS here; the API booster isn't capable of working with libc++ yet. - LLVM_CONFIG="${LLVM_ROOT}"/bin/llvm-config BAZEL_BUILD_OPTIONS="--config=clang" python3.8 ./tools/api_boost/api_boost_test.py + BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS[*]}" python3.8 ./tools/api_boost/api_boost_test.py exit 0 elif [[ "$CI_TARGET" == "bazel.coverage" || "$CI_TARGET" == "bazel.fuzz_coverage" ]]; then setup_clang_toolchain diff --git a/ci/go_mirror.sh b/ci/go_mirror.sh index 80be4cc0b5321..00e65816eb604 100755 --- a/ci/go_mirror.sh +++ b/ci/go_mirror.sh @@ -2,7 +2,11 @@ set -e -if [ -z "$CIRCLE_PULL_REQUEST" ] && [ "$CIRCLE_BRANCH" == "master" ] -then - tools/api/generate_go_protobuf.py +MAIN_BRANCH="refs/heads/azp_api" + +# shellcheck source=ci/setup_cache.sh +. "$(dirname "$0")"/setup_cache.sh + +if [[ "${AZP_BRANCH}" == "${MAIN_BRANCH}" ]]; then + BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_EXTRA_OPTIONS}" tools/api/generate_go_protobuf.py fi diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index 003a5ab28b76a..842b51b6ce894 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -56,6 +56,7 @@ mkdir -p "${ENVOY_DOCKER_BUILD_DIR}" [[ -t 1 ]] && ENVOY_DOCKER_OPTIONS+=("-it") [[ -f .git ]] && [[ ! -d .git ]] && ENVOY_DOCKER_OPTIONS+=(-v "$(git rev-parse --git-common-dir):$(git rev-parse --git-common-dir)") +[[ -n "${SSH_AUTH_SOCK}" ]] && ENVOY_DOCKER_OPTIONS+=(-v "${SSH_AUTH_SOCK}:${SSH_AUTH_SOCK}" -e SSH_AUTH_SOCK) export ENVOY_BUILD_IMAGE="${IMAGE_NAME}:${IMAGE_ID}" diff --git a/tools/api/generate_go_protobuf.py b/tools/api/generate_go_protobuf.py index 0cd15b6374493..c8867fef7515d 100755 --- a/tools/api/generate_go_protobuf.py +++ b/tools/api/generate_go_protobuf.py @@ -4,17 +4,21 @@ from subprocess import check_call import glob import os +import shlex import shutil import sys import re +# Needed for CI to pass down bazel options. +BAZEL_BUILD_OPTIONS = shlex.split(os.environ.get('BAZEL_BUILD_OPTIONS', '')) + TARGETS = '@envoy_api//...' IMPORT_BASE = 'github.com/envoyproxy/go-control-plane' OUTPUT_BASE = 'build_go' REPO_BASE = 'go-control-plane' -BRANCH = 'master' +BRANCH = 'master-ci-test' MIRROR_MSG = 'Mirrored from envoyproxy/envoy @ ' -USER_NAME = 'go-control-plane(CircleCI)' +USER_NAME = 'go-control-plane(Azure Pipelines)' USER_EMAIL = 'go-control-plane@users.noreply.github.com' @@ -32,7 +36,7 @@ def generateProtobufs(output): check_call([ 'bazel', 'build', '-c', 'fastbuild', '--experimental_proto_descriptor_sets_include_source_info' - ] + go_protos) + ] + BAZEL_BUILD_OPTIONS + go_protos) for rule in go_protos: # Example rule: @@ -67,9 +71,7 @@ def git(repo, *args): def cloneGoProtobufs(repo): # Create a local clone of go-control-plane - git(None, 'clone', 'git@github.com:envoyproxy/go-control-plane', repo) - git(repo, 'fetch') - git(repo, 'checkout', '-B', BRANCH, 'origin/master') + git(None, 'clone', 'git@github.com:envoyproxy/go-control-plane', repo, '-b', BRANCH) def findLastSyncSHA(repo): diff --git a/tools/api_boost/api_boost.py b/tools/api_boost/api_boost.py index eda6eaf940881..5cd9846bcf218 100755 --- a/tools/api_boost/api_boost.py +++ b/tools/api_boost/api_boost.py @@ -132,7 +132,6 @@ def ApiBoostTree(target_paths, sp.run([ 'bazel', 'build', - '--config=libc++', '--strip=always', ] + BAZEL_BUILD_OPTIONS + dep_lib_build_targets, check=True) From 0b5bceb856af0e9e912d3032194fc09cb3d6ad2f Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Tue, 13 Oct 2020 15:38:45 -0700 Subject: [PATCH 2/7] use main branch Signed-off-by: Lizan Zhou --- .azure-pipelines/pipelines.yml | 6 ++--- .circleci/config.yml | 43 ------------------------------- ci/api_mirror.sh | 4 +-- ci/go_mirror.sh | 2 +- tools/api/generate_go_protobuf.py | 2 +- 5 files changed, 6 insertions(+), 51 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml index 0b46ca296a17f..f8c5ce9721740 100644 --- a/.azure-pipelines/pipelines.yml +++ b/.azure-pipelines/pipelines.yml @@ -138,6 +138,7 @@ jobs: - job: api displayName: "data-plane-api sync" dependsOn: [] + condition: and(succeeded(), eq(variables['PostSubmit'], true)) pool: vmImage: "ubuntu-18.04" steps: @@ -147,18 +148,17 @@ jobs: sshPublicKey: "$(DataPlaneApiPublicKey)" sshPassphrase: "$(SshDeployKeyPassphrase)" sshKeySecureFile: "$(DataPlaneApiPrivateKey)" - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true)) - bash: ci/api_mirror.sh displayName: "Sync data-plane-api" workingDirectory: $(Build.SourcesDirectory) env: AZP_BRANCH: $(Build.SourceBranch) - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true)) - job: go_control_plane displayName: "go-control-plane sync" dependsOn: [] + condition: and(succeeded(), eq(variables['PostSubmit'], true)) steps: - task: InstallSSHKey@0 inputs: @@ -166,7 +166,6 @@ jobs: sshPublicKey: "$(GoControlPlanePublicKey)" sshPassphrase: "$(SshDeployKeyPassphrase)" sshKeySecureFile: "$(GoControlPlanePrivateKey)" - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true)) - bash: | cp -a ~/.ssh $(Build.StagingDirectory)/ @@ -179,7 +178,6 @@ jobs: BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) AZP_BRANCH: $(Build.SourceBranch) - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true)) - job: bazel displayName: "Linux-x64" diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index e04a9737d8f91..0000000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,43 +0,0 @@ -version: 2.1 - -executors: - ubuntu-build: - description: "A regular build executor based on ubuntu image" - docker: - # NOTE: Update this from https://github.com/envoyproxy/envoy-build-tools/blob/master/toolchains/rbe_toolchains_config.bzl#L8 - - image: envoyproxy/envoy-build-ubuntu:b480535e8423b5fd7c102fd30c92f4785519e33a - resource_class: xlarge - working_directory: /source - -jobs: - api: - executor: ubuntu-build - steps: - - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken - - checkout - - run: ci/do_circle_ci.sh bazel.api - - add_ssh_keys: - fingerprints: - - "fb:f3:fe:be:1c:b2:ec:b6:25:f9:7b:a6:87:54:02:8c" - - run: ci/api_mirror.sh - - store_artifacts: - path: /build/envoy/generated - destination: / - - go_control_plane_mirror: - executor: ubuntu-build - steps: - - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken - - checkout - - run: ci/do_circle_ci.sh bazel.api - - add_ssh_keys: - fingerprints: - - "9d:3b:fe:7c:09:3b:ce:a9:6a:de:de:41:fb:6b:52:62" - - run: ci/go_mirror.sh - -workflows: - version: 2 - all: - jobs: - - api - - go_control_plane_mirror diff --git a/ci/api_mirror.sh b/ci/api_mirror.sh index 09868f72f1d30..03e8ab85d80cb 100755 --- a/ci/api_mirror.sh +++ b/ci/api_mirror.sh @@ -3,8 +3,8 @@ set -e CHECKOUT_DIR=../data-plane-api -MAIN_BRANCH="refs/heads/azp_api" -API_MAIN_BRANCH="master-ci-test" +MAIN_BRANCH="refs/heads/master" +API_MAIN_BRANCH="master" if [[ "${AZP_BRANCH}" == "${MAIN_BRANCH}" ]]; then echo "Cloning..." diff --git a/ci/go_mirror.sh b/ci/go_mirror.sh index 00e65816eb604..63f96d0d79697 100755 --- a/ci/go_mirror.sh +++ b/ci/go_mirror.sh @@ -2,7 +2,7 @@ set -e -MAIN_BRANCH="refs/heads/azp_api" +MAIN_BRANCH="refs/heads/master" # shellcheck source=ci/setup_cache.sh . "$(dirname "$0")"/setup_cache.sh diff --git a/tools/api/generate_go_protobuf.py b/tools/api/generate_go_protobuf.py index c8867fef7515d..5b25de2dbb0a4 100755 --- a/tools/api/generate_go_protobuf.py +++ b/tools/api/generate_go_protobuf.py @@ -16,7 +16,7 @@ IMPORT_BASE = 'github.com/envoyproxy/go-control-plane' OUTPUT_BASE = 'build_go' REPO_BASE = 'go-control-plane' -BRANCH = 'master-ci-test' +BRANCH = 'master' MIRROR_MSG = 'Mirrored from envoyproxy/envoy @ ' USER_NAME = 'go-control-plane(Azure Pipelines)' USER_EMAIL = 'go-control-plane@users.noreply.github.com' From b459461aec995dd145afe780643565310c979215 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Tue, 13 Oct 2020 21:03:35 -0700 Subject: [PATCH 3/7] comment Signed-off-by: Lizan Zhou --- ci/do_ci.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 3e9eb939ae845..f3958aeaedf6d 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -312,7 +312,8 @@ elif [[ "$CI_TARGET" == "bazel.compile_time_options" ]]; then collect_build_profile build exit 0 elif [[ "$CI_TARGET" == "bazel.api" ]]; then - # Use libstdc++ because the API booster isn't capable of working with libc++ yet. + # Use libstdc++ because the API booster links to prebuilt libclang*/libLLVM* installed in /opt/llvm/lib, + # which is built with libstdc++. Using libstdc++ for whole of the API CI job to avoid unnecessary rebuild. ENVOY_STDLIB="libstdc++" setup_clang_toolchain export LLVM_CONFIG="${LLVM_ROOT}"/bin/llvm-config From 27bf2d75701cf6c10f22339ca03cc82ed55aa164 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Wed, 14 Oct 2020 00:11:17 -0700 Subject: [PATCH 4/7] ci: use multiple stage Signed-off-by: Lizan Zhou --- .azure-pipelines/pipelines.yml | 692 +++++++++++++++++---------------- 1 file changed, 353 insertions(+), 339 deletions(-) diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml index f8c5ce9721740..ffd5ed176c836 100644 --- a/.azure-pipelines/pipelines.yml +++ b/.azure-pipelines/pipelines.yml @@ -10,370 +10,384 @@ trigger: # PR build config is manually overridden in Azure pipelines UI with different secrets pr: none -jobs: - - job: format - dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel. - pool: - vmImage: "ubuntu-18.04" - steps: - - task: Cache@2 - inputs: - key: "format | ./WORKSPACE | **/*.bzl" - path: $(Build.StagingDirectory)/repository_cache - continueOnError: true +stages: + - stage: precheck + jobs: + - job: format + dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel. + pool: + vmImage: "ubuntu-18.04" + steps: + - task: Cache@2 + inputs: + key: "format | ./WORKSPACE | **/*.bzl" + path: $(Build.StagingDirectory)/repository_cache + continueOnError: true - - script: ci/run_envoy_docker.sh 'ci/check_and_fix_format.sh' - workingDirectory: $(Build.SourcesDirectory) - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - displayName: "Run check format scripts" + - script: ci/run_envoy_docker.sh 'ci/check_and_fix_format.sh' + workingDirectory: $(Build.SourcesDirectory) + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + displayName: "Run check format scripts" - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: "$(Build.StagingDirectory)/fix_format.diff" - artifactName: format - condition: failed() + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: "$(Build.StagingDirectory)/fix_format.diff" + artifactName: format + condition: failed() - - job: docs - dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel. - pool: - vmImage: "ubuntu-18.04" - steps: - - task: Cache@2 - inputs: - key: "docs | ./WORKSPACE | **/*.bzl" - path: $(Build.StagingDirectory)/repository_cache - continueOnError: true + - job: docs + dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel. + pool: + vmImage: "ubuntu-18.04" + steps: + - task: Cache@2 + inputs: + key: "docs | ./WORKSPACE | **/*.bzl" + path: $(Build.StagingDirectory)/repository_cache + continueOnError: true - - script: ci/run_envoy_docker.sh 'ci/do_ci.sh docs' - workingDirectory: $(Build.SourcesDirectory) - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - displayName: "Generate docs" + - script: ci/run_envoy_docker.sh 'ci/do_ci.sh docs' + workingDirectory: $(Build.SourcesDirectory) + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + displayName: "Generate docs" - - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/docs docs' - displayName: "Upload Docs to GCS" - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket) + - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/docs docs' + displayName: "Upload Docs to GCS" + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket) - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: "$(Build.SourcesDirectory)/generated/docs" - artifactName: docs - condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: "$(Build.SourcesDirectory)/generated/docs" + artifactName: docs + condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) - - task: InstallSSHKey@0 - inputs: - hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" - sshPublicKey: "$(DocsPublicKey)" - sshPassphrase: "$(SshDeployKeyPassphrase)" - sshKeySecureFile: "$(DocsPrivateKey)" - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true)) + - task: InstallSSHKey@0 + inputs: + hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" + sshPublicKey: "$(DocsPublicKey)" + sshPassphrase: "$(SshDeployKeyPassphrase)" + sshKeySecureFile: "$(DocsPrivateKey)" + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true)) - - script: docs/publish.sh - displayName: "Publish to GitHub" - workingDirectory: $(Build.SourcesDirectory) - env: - AZP_BRANCH: $(Build.SourceBranch) - AZP_SHA1: $(Build.SourceVersion) - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true)) + - script: docs/publish.sh + displayName: "Publish to GitHub" + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) + AZP_SHA1: $(Build.SourceVersion) + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true)) - - job: release - displayName: "Linux-x64 release" - dependsOn: ["format"] - # For master builds, continue even if format fails - condition: and(not(canceled()), or(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))) - timeoutInMinutes: 360 - pool: - vmImage: "ubuntu-18.04" - steps: - - template: bazel.yml - parameters: - ciTarget: bazel.release - - - job: release_arm64 - displayName: "Linux-arm64 release" - dependsOn: ["format"] - # For master builds, continue even if format fails - condition: and(not(canceled()), or(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))) - timeoutInMinutes: 360 - pool: "arm-large" - steps: - - template: bazel.yml - parameters: - managedAgent: false - ciTarget: bazel.release - rbe: false - artifactSuffix: ".arm64" - bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base" - - - job: filter_example - displayName: "filter-example sync" - dependsOn: [] - pool: - vmImage: "ubuntu-18.04" + - stage: sync condition: and(succeeded(), eq(variables['PostSubmit'], true)) - steps: - - task: InstallSSHKey@0 - inputs: - hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" - sshPublicKey: "$(FilterExamplePublicKey)" - sshPassphrase: "$(SshDeployKeyPassphrase)" - sshKeySecureFile: "$(FilterExamplePrivateKey)" + dependsOn: [] + jobs: + - job: filter_example + displayName: "filter-example sync" + dependsOn: [] + pool: + vmImage: "ubuntu-18.04" + steps: + - task: InstallSSHKey@0 + inputs: + hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" + sshPublicKey: "$(FilterExamplePublicKey)" + sshPassphrase: "$(SshDeployKeyPassphrase)" + sshKeySecureFile: "$(FilterExamplePrivateKey)" - - bash: ci/filter_example_mirror.sh - displayName: "Sync envoy-filter-example" - workingDirectory: $(Build.SourcesDirectory) - env: - AZP_BRANCH: $(Build.SourceBranch) + - bash: ci/filter_example_mirror.sh + displayName: "Sync envoy-filter-example" + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) - - job: api - displayName: "data-plane-api sync" - dependsOn: [] - condition: and(succeeded(), eq(variables['PostSubmit'], true)) - pool: - vmImage: "ubuntu-18.04" - steps: - - task: InstallSSHKey@0 - inputs: - hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" - sshPublicKey: "$(DataPlaneApiPublicKey)" - sshPassphrase: "$(SshDeployKeyPassphrase)" - sshKeySecureFile: "$(DataPlaneApiPrivateKey)" + - job: api + displayName: "data-plane-api sync" + dependsOn: [] + pool: + vmImage: "ubuntu-18.04" + steps: + - task: InstallSSHKey@0 + inputs: + hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" + sshPublicKey: "$(DataPlaneApiPublicKey)" + sshPassphrase: "$(SshDeployKeyPassphrase)" + sshKeySecureFile: "$(DataPlaneApiPrivateKey)" - - bash: ci/api_mirror.sh - displayName: "Sync data-plane-api" - workingDirectory: $(Build.SourcesDirectory) - env: - AZP_BRANCH: $(Build.SourceBranch) + - bash: ci/api_mirror.sh + displayName: "Sync data-plane-api" + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) - - job: go_control_plane - displayName: "go-control-plane sync" - dependsOn: [] - condition: and(succeeded(), eq(variables['PostSubmit'], true)) - steps: - - task: InstallSSHKey@0 - inputs: - hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" - sshPublicKey: "$(GoControlPlanePublicKey)" - sshPassphrase: "$(SshDeployKeyPassphrase)" - sshKeySecureFile: "$(GoControlPlanePrivateKey)" + - job: go_control_plane + displayName: "go-control-plane sync" + dependsOn: [] + steps: + - task: InstallSSHKey@0 + inputs: + hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" + sshPublicKey: "$(GoControlPlanePublicKey)" + sshPassphrase: "$(SshDeployKeyPassphrase)" + sshKeySecureFile: "$(GoControlPlanePrivateKey)" - - bash: | - cp -a ~/.ssh $(Build.StagingDirectory)/ - ci/run_envoy_docker.sh 'ci/go_mirror.sh' - displayName: "Sync go-control-plane" - workingDirectory: $(Build.SourcesDirectory) - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - AZP_BRANCH: $(Build.SourceBranch) + - bash: | + cp -a ~/.ssh $(Build.StagingDirectory)/ + ci/run_envoy_docker.sh 'ci/go_mirror.sh' + displayName: "Sync go-control-plane" + workingDirectory: $(Build.SourcesDirectory) + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + AZP_BRANCH: $(Build.SourceBranch) - - job: bazel - displayName: "Linux-x64" - dependsOn: ["release"] - # For master builds, continue even if format fails + - stage: linux_x64 + dependsOn: ["precheck"] + # For master builds, continue even if precheck fails condition: and(not(canceled()), or(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))) - strategy: - maxParallel: 3 - matrix: - api: - CI_TARGET: "bazel.api" - gcc: - CI_TARGET: "bazel.gcc" - clang_tidy: - CI_TARGET: "bazel.clang_tidy" - asan: - CI_TARGET: "bazel.asan" - tsan: - CI_TARGET: "bazel.tsan" - compile_time_options: - CI_TARGET: "bazel.compile_time_options" - timeoutInMinutes: 360 - pool: - vmImage: "ubuntu-18.04" - steps: - - template: bazel.yml - parameters: - ciTarget: $(CI_TARGET) + jobs: + - job: release + # For master builds, continue even if format fails + timeoutInMinutes: 360 + pool: + vmImage: "ubuntu-18.04" + steps: + - template: bazel.yml + parameters: + ciTarget: bazel.release + + - stage: linux_arm64 + dependsOn: ["precheck"] + # For master builds, continue even if precheck fails + condition: and(not(canceled()), or(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))) + jobs: + - job: release + timeoutInMinutes: 360 + pool: "arm-large" + steps: + - template: bazel.yml + parameters: + managedAgent: false + ciTarget: bazel.release + rbe: false + artifactSuffix: ".arm64" + bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base" + + - stage: check + dependsOn: ["linux_x64"] + jobs: + - job: bazel + displayName: "Linux-x64" + dependsOn: [] + strategy: + maxParallel: 3 + matrix: + api: + CI_TARGET: "bazel.api" + gcc: + CI_TARGET: "bazel.gcc" + clang_tidy: + CI_TARGET: "bazel.clang_tidy" + asan: + CI_TARGET: "bazel.asan" + tsan: + CI_TARGET: "bazel.tsan" + compile_time_options: + CI_TARGET: "bazel.compile_time_options" + timeoutInMinutes: 360 + pool: + vmImage: "ubuntu-18.04" + steps: + - template: bazel.yml + parameters: + ciTarget: $(CI_TARGET) - - job: coverage - displayName: "Linux-x64" - dependsOn: ["release"] - timeoutInMinutes: 360 - pool: "x64-large" - strategy: - maxParallel: 2 - matrix: - coverage: - CI_TARGET: "coverage" - fuzz_coverage: - CI_TARGET: "fuzz_coverage" - steps: - - template: bazel.yml - parameters: - managedAgent: false - ciTarget: bazel.$(CI_TARGET) - rbe: false - # /tmp/sandbox_base is a tmpfs in CI environment to optimize large I/O for coverage traces - bazelBuildExtraOptions: "--define=no_debug_info=1 --linkopt=-Wl,-s --test_env=ENVOY_IP_TEST_VERSIONS=v4only --sandbox_base=/tmp/sandbox_base" + - job: coverage + displayName: "Linux-x64" + dependsOn: [] + timeoutInMinutes: 360 + pool: "x64-large" + strategy: + maxParallel: 2 + matrix: + coverage: + CI_TARGET: "coverage" + fuzz_coverage: + CI_TARGET: "fuzz_coverage" + steps: + - template: bazel.yml + parameters: + managedAgent: false + ciTarget: bazel.$(CI_TARGET) + rbe: false + # /tmp/sandbox_base is a tmpfs in CI environment to optimize large I/O for coverage traces + bazelBuildExtraOptions: "--define=no_debug_info=1 --linkopt=-Wl,-s --test_env=ENVOY_IP_TEST_VERSIONS=v4only --sandbox_base=/tmp/sandbox_base" - - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/$(CI_TARGET) $(CI_TARGET)' - displayName: "Upload $(CI_TARGET) Report to GCS" - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket) - condition: always() + - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/$(CI_TARGET) $(CI_TARGET)' + displayName: "Upload $(CI_TARGET) Report to GCS" + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket) + condition: always() - - job: docker - displayName: "Linux multi-arch docker" - dependsOn: ["release", "release_arm64"] - pool: - vmImage: "ubuntu-18.04" - steps: - - bash: .azure-pipelines/cleanup.sh - displayName: "Removing tools from agent" - - task: DownloadBuildArtifacts@0 - inputs: - buildType: current - artifactName: "bazel.release" - itemPattern: "bazel.release/envoy_binary.tar.gz" - downloadType: single - targetPath: $(Build.StagingDirectory) - - task: DownloadBuildArtifacts@0 - inputs: - buildType: current - artifactName: "bazel.release.arm64" - itemPattern: "bazel.release.arm64/envoy_binary.tar.gz" - downloadType: single - targetPath: $(Build.StagingDirectory) - - bash: | - set -e - mkdir -p linux/amd64 && tar zxf $(Build.StagingDirectory)/bazel.release/envoy_binary.tar.gz -C ./linux/amd64 - mkdir -p linux/arm64 && tar zxf $(Build.StagingDirectory)/bazel.release.arm64/envoy_binary.tar.gz -C ./linux/arm64 - ci/docker_ci.sh - workingDirectory: $(Build.SourcesDirectory) - env: - AZP_BRANCH: $(Build.SourceBranch) - AZP_SHA1: $(Build.SourceVersion) - DOCKERHUB_USERNAME: $(DockerUsername) - DOCKERHUB_PASSWORD: $(DockerPassword) - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: "$(Build.StagingDirectory)/build_images" - artifactName: docker - condition: always() + - stage: docker + dependsOn: ["linux_x64", "linux_arm64"] + jobs: + - job: docker + displayName: "Linux multi-arch docker" + pool: + vmImage: "ubuntu-18.04" + steps: + - bash: .azure-pipelines/cleanup.sh + displayName: "Removing tools from agent" + - task: DownloadBuildArtifacts@0 + inputs: + buildType: current + artifactName: "bazel.release" + itemPattern: "bazel.release/envoy_binary.tar.gz" + downloadType: single + targetPath: $(Build.StagingDirectory) + - task: DownloadBuildArtifacts@0 + inputs: + buildType: current + artifactName: "bazel.release.arm64" + itemPattern: "bazel.release.arm64/envoy_binary.tar.gz" + downloadType: single + targetPath: $(Build.StagingDirectory) + - bash: | + set -e + mkdir -p linux/amd64 && tar zxf $(Build.StagingDirectory)/bazel.release/envoy_binary.tar.gz -C ./linux/amd64 + mkdir -p linux/arm64 && tar zxf $(Build.StagingDirectory)/bazel.release.arm64/envoy_binary.tar.gz -C ./linux/arm64 + ci/docker_ci.sh + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) + AZP_SHA1: $(Build.SourceVersion) + DOCKERHUB_USERNAME: $(DockerUsername) + DOCKERHUB_PASSWORD: $(DockerPassword) + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: "$(Build.StagingDirectory)/build_images" + artifactName: docker + condition: always() - - job: examples + - stage: verify dependsOn: ["docker"] - displayName: "Verify examples run as documented" - pool: - vmImage: "ubuntu-18.04" - steps: - - task: DownloadBuildArtifacts@0 - inputs: - buildType: current - artifactName: "docker" - itemPattern: "docker/envoy-docker-images.tar.xz" - downloadType: single - targetPath: $(Build.StagingDirectory) - - bash: ./ci/do_ci.sh verify_examples - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - NO_BUILD_SETUP: 1 + jobs: + - job: examples + displayName: "Verify examples run as documented" + pool: + vmImage: "ubuntu-18.04" + steps: + - task: DownloadBuildArtifacts@0 + inputs: + buildType: current + artifactName: "docker" + itemPattern: "docker/envoy-docker-images.tar.xz" + downloadType: single + targetPath: $(Build.StagingDirectory) + - bash: ./ci/do_ci.sh verify_examples + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + NO_BUILD_SETUP: 1 - - job: macOS - dependsOn: ["format"] - timeoutInMinutes: 360 - pool: - vmImage: "macos-latest" - steps: - - script: ./ci/mac_ci_setup.sh - displayName: "Install dependencies" + - stage: macos + dependsOn: ["precheck"] + jobs: + - job: test + timeoutInMinutes: 360 + pool: + vmImage: "macos-latest" + steps: + - script: ./ci/mac_ci_setup.sh + displayName: "Install dependencies" - - script: ./ci/mac_ci_steps.sh - displayName: "Run Mac CI" - env: - BAZEL_BUILD_EXTRA_OPTIONS: "--remote_download_toplevel --flaky_test_attempts=2" - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + - script: ./ci/mac_ci_steps.sh + displayName: "Run Mac CI" + env: + BAZEL_BUILD_EXTRA_OPTIONS: "--remote_download_toplevel --flaky_test_attempts=2" + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - - task: PublishTestResults@2 - inputs: - testResultsFiles: "**/bazel-testlogs/**/test.xml" - testRunTitle: "macOS" - condition: always() + - task: PublishTestResults@2 + inputs: + testResultsFiles: "**/bazel-testlogs/**/test.xml" + testRunTitle: "macOS" + condition: always() - - script: ./ci/flaky_test/run_process_xml.sh - displayName: "Process Test Results" - env: - TEST_TMPDIR: $(Build.SourcesDirectory) - SLACK_TOKEN: $(SLACK_TOKEN) - CI_TARGET: "MacOS" - REPO_URI: $(Build.Repository.Uri) - BUILD_URI: $(Build.BuildUri) + - script: ./ci/flaky_test/run_process_xml.sh + displayName: "Process Test Results" + env: + TEST_TMPDIR: $(Build.SourcesDirectory) + SLACK_TOKEN: $(SLACK_TOKEN) + CI_TARGET: "MacOS" + REPO_URI: $(Build.Repository.Uri) + BUILD_URI: $(Build.BuildUri) - - job: Windows - dependsOn: ["format"] - timeoutInMinutes: 360 - pool: - vmImage: "windows-latest" - steps: - - bash: ci/run_envoy_docker.sh ci/windows_ci_steps.sh - displayName: "Run Windows CI" - env: - ENVOY_DOCKER_BUILD_DIR: "$(Build.StagingDirectory)" - ENVOY_RBE: "true" - BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=remote-msvc-cl --jobs=$(RbeJobs)" - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: "$(Build.StagingDirectory)/envoy" - artifactName: windows.release - condition: always() + - stage: windows + dependsOn: ["precheck"] + jobs: + - job: release + timeoutInMinutes: 360 + pool: + vmImage: "windows-latest" + steps: + - bash: ci/run_envoy_docker.sh ci/windows_ci_steps.sh + displayName: "Run Windows CI" + env: + ENVOY_DOCKER_BUILD_DIR: "$(Build.StagingDirectory)" + ENVOY_RBE: "true" + BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=remote-msvc-cl --jobs=$(RbeJobs)" + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: "$(Build.StagingDirectory)/envoy" + artifactName: windows.release + condition: always() - - job: docker_windows - displayName: "Windows docker" - dependsOn: ["Windows"] - timeoutInMinutes: 360 - pool: - vmImage: "windows-latest" - steps: - - task: DownloadBuildArtifacts@0 - inputs: - buildType: current - artifactName: "windows.release" - itemPattern: "windows.release/envoy_binary.tar.gz" - downloadType: single - targetPath: $(Build.StagingDirectory) - - bash: | - set -e - # Convert to Unix-style path so tar doesn't think drive letter is a hostname - STAGING_DIR="/$(echo '$(Build.StagingDirectory)' | tr -d ':' | tr '\\' '/')" - mkdir -p windows/amd64 && tar zxf "${STAGING_DIR}/windows.release/envoy_binary.tar.gz" -C ./windows/amd64 - ci/docker_ci.sh - workingDirectory: $(Build.SourcesDirectory) - env: - AZP_BRANCH: $(Build.SourceBranch) - AZP_SHA1: $(Build.SourceVersion) - DOCKERHUB_USERNAME: $(DockerUsername) - DOCKERHUB_PASSWORD: $(DockerPassword) - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: "$(Build.StagingDirectory)/build_images" - artifactName: docker_windows - condition: always() + - job: docker + dependsOn: ["release"] + timeoutInMinutes: 360 + pool: + vmImage: "windows-latest" + steps: + - task: DownloadBuildArtifacts@0 + inputs: + buildType: current + artifactName: "windows.release" + itemPattern: "windows.release/envoy_binary.tar.gz" + downloadType: single + targetPath: $(Build.StagingDirectory) + - bash: | + set -e + # Convert to Unix-style path so tar doesn't think drive letter is a hostname + STAGING_DIR="/$(echo '$(Build.StagingDirectory)' | tr -d ':' | tr '\\' '/')" + mkdir -p windows/amd64 && tar zxf "${STAGING_DIR}/windows.release/envoy_binary.tar.gz" -C ./windows/amd64 + ci/docker_ci.sh + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) + AZP_SHA1: $(Build.SourceVersion) + DOCKERHUB_USERNAME: $(DockerUsername) + DOCKERHUB_PASSWORD: $(DockerPassword) + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: "$(Build.StagingDirectory)/build_images" + artifactName: docker_windows + condition: always() From f3392df5a4fd3ecf7a8df6863ddfa5b99f973662 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Wed, 14 Oct 2020 11:46:38 -0700 Subject: [PATCH 5/7] cosmetic Signed-off-by: Lizan Zhou --- .azure-pipelines/pipelines.yml | 95 +--------------------------------- 1 file changed, 1 insertion(+), 94 deletions(-) diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml index faa223c36657c..66f2a3838b750 100644 --- a/.azure-pipelines/pipelines.yml +++ b/.azure-pipelines/pipelines.yml @@ -93,7 +93,6 @@ stages: dependsOn: [] jobs: - job: filter_example - displayName: "filter-example sync" dependsOn: [] pool: vmImage: "ubuntu-18.04" @@ -111,9 +110,7 @@ stages: env: AZP_BRANCH: $(Build.SourceBranch) -<<<<<<< HEAD - - job: api - displayName: "data-plane-api sync" + - job: data_plane_api dependsOn: [] pool: vmImage: "ubuntu-18.04" @@ -124,21 +121,6 @@ stages: sshPublicKey: "$(DataPlaneApiPublicKey)" sshPassphrase: "$(SshDeployKeyPassphrase)" sshKeySecureFile: "$(DataPlaneApiPrivateKey)" -======= - - job: filter_example - displayName: "filter-example sync" - dependsOn: [] - pool: - vmImage: "ubuntu-18.04" - condition: and(succeeded(), eq(variables['PostSubmit'], true)) - steps: - - task: InstallSSHKey@0 - inputs: - hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" - sshPublicKey: "$(FilterExamplePublicKey)" - sshPassphrase: "$(SshDeployKeyPassphrase)" - sshKeySecureFile: "$(FilterExamplePrivateKey)" ->>>>>>> upstream/master - bash: ci/api_mirror.sh displayName: "Sync data-plane-api" @@ -146,9 +128,7 @@ stages: env: AZP_BRANCH: $(Build.SourceBranch) -<<<<<<< HEAD - job: go_control_plane - displayName: "go-control-plane sync" dependsOn: [] steps: - task: InstallSSHKey@0 @@ -230,79 +210,6 @@ stages: - template: bazel.yml parameters: ciTarget: $(CI_TARGET) -======= - - job: api - displayName: "data-plane-api sync" - dependsOn: [] - condition: and(succeeded(), eq(variables['PostSubmit'], true)) - pool: - vmImage: "ubuntu-18.04" - steps: - - task: InstallSSHKey@0 - inputs: - hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" - sshPublicKey: "$(DataPlaneApiPublicKey)" - sshPassphrase: "$(SshDeployKeyPassphrase)" - sshKeySecureFile: "$(DataPlaneApiPrivateKey)" - - - bash: ci/api_mirror.sh - displayName: "Sync data-plane-api" - workingDirectory: $(Build.SourcesDirectory) - env: - AZP_BRANCH: $(Build.SourceBranch) - - - job: go_control_plane - displayName: "go-control-plane sync" - dependsOn: [] - condition: and(succeeded(), eq(variables['PostSubmit'], true)) - steps: - - task: InstallSSHKey@0 - inputs: - hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" - sshPublicKey: "$(GoControlPlanePublicKey)" - sshPassphrase: "$(SshDeployKeyPassphrase)" - sshKeySecureFile: "$(GoControlPlanePrivateKey)" - - - bash: | - cp -a ~/.ssh $(Build.StagingDirectory)/ - ci/run_envoy_docker.sh 'ci/go_mirror.sh' - displayName: "Sync go-control-plane" - workingDirectory: $(Build.SourcesDirectory) - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - AZP_BRANCH: $(Build.SourceBranch) - - - job: bazel - displayName: "Linux-x64" - dependsOn: ["release"] - # For master builds, continue even if format fails - condition: and(not(canceled()), or(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))) - strategy: - maxParallel: 3 - matrix: - api: - CI_TARGET: "bazel.api" - gcc: - CI_TARGET: "bazel.gcc" - clang_tidy: - CI_TARGET: "bazel.clang_tidy" - asan: - CI_TARGET: "bazel.asan" - tsan: - CI_TARGET: "bazel.tsan" - compile_time_options: - CI_TARGET: "bazel.compile_time_options" - timeoutInMinutes: 360 - pool: - vmImage: "ubuntu-18.04" - steps: - - template: bazel.yml - parameters: - ciTarget: $(CI_TARGET) ->>>>>>> upstream/master - job: coverage displayName: "Linux-x64" From 80078f0d07830ca9e9e5d72e4e2d47309e060c90 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Wed, 14 Oct 2020 17:39:51 -0700 Subject: [PATCH 6/7] OCD Signed-off-by: Lizan Zhou --- .azure-pipelines/pipelines.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml index 66f2a3838b750..f8bd6baaf879c 100644 --- a/.azure-pipelines/pipelines.yml +++ b/.azure-pipelines/pipelines.yml @@ -186,7 +186,7 @@ stages: dependsOn: ["linux_x64"] jobs: - job: bazel - displayName: "Linux-x64" + displayName: "linux_x64" dependsOn: [] strategy: maxParallel: 3 @@ -212,7 +212,7 @@ stages: ciTarget: $(CI_TARGET) - job: coverage - displayName: "Linux-x64" + displayName: "linux_x64" dependsOn: [] timeoutInMinutes: 360 pool: "x64-large" @@ -285,7 +285,6 @@ stages: dependsOn: ["docker"] jobs: - job: examples - displayName: "Verify examples run as documented" pool: vmImage: "ubuntu-18.04" steps: From 265a141453833f6c30dfbb6b9affac9f0d55b40f Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Wed, 14 Oct 2020 21:25:14 -0700 Subject: [PATCH 7/7] another OCD Signed-off-by: Lizan Zhou --- .azure-pipelines/pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml index f8bd6baaf879c..151736a8f335e 100644 --- a/.azure-pipelines/pipelines.yml +++ b/.azure-pipelines/pipelines.yml @@ -244,7 +244,7 @@ stages: dependsOn: ["linux_x64", "linux_arm64"] jobs: - job: docker - displayName: "Linux multi-arch docker" + displayName: "linux multiarch" pool: vmImage: "ubuntu-18.04" steps: