From 79c2292530cb1772c232ed08b3cc47675c7ed996 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Mon, 12 Oct 2020 21:36:10 -0700 Subject: [PATCH 1/3] ci: use azp for api and go-control-plane sync Signed-off-by: Lizan Zhou --- .azure-pipelines/pipelines.yml | 50 +++++++++++++++++++++++++++++++ ci/api_mirror.sh | 13 ++++---- ci/do_ci.sh | 13 ++++---- ci/go_mirror.sh | 10 +++++-- ci/run_envoy_docker.sh | 1 + tools/api/generate_go_protobuf.py | 14 +++++---- tools/api_boost/api_boost.py | 1 - 7 files changed, 79 insertions(+), 23 deletions(-) diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml index 31fb5e06fae01..0b46ca296a17f 100644 --- a/.azure-pipelines/pipelines.yml +++ b/.azure-pipelines/pipelines.yml @@ -118,6 +118,8 @@ jobs: - job: filter_example displayName: "filter-example sync" dependsOn: [] + pool: + vmImage: "ubuntu-18.04" condition: and(succeeded(), eq(variables['PostSubmit'], true)) steps: - task: InstallSSHKey@0 @@ -133,6 +135,52 @@ jobs: env: AZP_BRANCH: $(Build.SourceBranch) + - job: api + displayName: "data-plane-api sync" + dependsOn: [] + pool: + vmImage: "ubuntu-18.04" + steps: + - task: InstallSSHKey@0 + inputs: + hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" + sshPublicKey: "$(DataPlaneApiPublicKey)" + sshPassphrase: "$(SshDeployKeyPassphrase)" + sshKeySecureFile: "$(DataPlaneApiPrivateKey)" + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true)) + + - bash: ci/api_mirror.sh + displayName: "Sync data-plane-api" + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true)) + + - job: go_control_plane + displayName: "go-control-plane sync" + dependsOn: [] + steps: + - task: InstallSSHKey@0 + inputs: + hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" + sshPublicKey: "$(GoControlPlanePublicKey)" + sshPassphrase: "$(SshDeployKeyPassphrase)" + sshKeySecureFile: "$(GoControlPlanePrivateKey)" + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true)) + + - bash: | + cp -a ~/.ssh $(Build.StagingDirectory)/ + ci/run_envoy_docker.sh 'ci/go_mirror.sh' + displayName: "Sync go-control-plane" + workingDirectory: $(Build.SourcesDirectory) + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + AZP_BRANCH: $(Build.SourceBranch) + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true)) + - job: bazel displayName: "Linux-x64" dependsOn: ["release"] @@ -141,6 +189,8 @@ jobs: strategy: maxParallel: 3 matrix: + api: + CI_TARGET: "bazel.api" gcc: CI_TARGET: "bazel.gcc" clang_tidy: diff --git a/ci/api_mirror.sh b/ci/api_mirror.sh index 077cdd1d3cfe6..09868f72f1d30 100755 --- a/ci/api_mirror.sh +++ b/ci/api_mirror.sh @@ -3,16 +3,15 @@ set -e CHECKOUT_DIR=../data-plane-api +MAIN_BRANCH="refs/heads/azp_api" +API_MAIN_BRANCH="master-ci-test" -if [ -z "$CIRCLE_PULL_REQUEST" ] && [ "$CIRCLE_BRANCH" == "master" ] -then +if [[ "${AZP_BRANCH}" == "${MAIN_BRANCH}" ]]; then echo "Cloning..." - git clone git@github.com:envoyproxy/data-plane-api "$CHECKOUT_DIR" + git clone git@github.com:envoyproxy/data-plane-api "$CHECKOUT_DIR" -b "${API_MAIN_BRANCH}" - git -C "$CHECKOUT_DIR" config user.name "data-plane-api(CircleCI)" + git -C "$CHECKOUT_DIR" config user.name "data-plane-api(Azure Pipelines)" git -C "$CHECKOUT_DIR" config user.email data-plane-api@users.noreply.github.com - git -C "$CHECKOUT_DIR" fetch - git -C "$CHECKOUT_DIR" checkout -B master origin/master # Determine last envoyproxy/envoy SHA in envoyproxy/data-plane-api MIRROR_MSG="Mirrored from https://github.com/envoyproxy/envoy" @@ -40,6 +39,6 @@ then done echo "Pushing..." - git -C "$CHECKOUT_DIR" push origin master + git -C "$CHECKOUT_DIR" push origin "${API_MAIN_BRANCH}" echo "Done" fi diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 4e908fd6aab48..3e9eb939ae845 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -312,19 +312,20 @@ elif [[ "$CI_TARGET" == "bazel.compile_time_options" ]]; then collect_build_profile build exit 0 elif [[ "$CI_TARGET" == "bazel.api" ]]; then + # Use libstdc++ because the API booster isn't capable of working with libc++ yet. + ENVOY_STDLIB="libstdc++" setup_clang_toolchain + export LLVM_CONFIG="${LLVM_ROOT}"/bin/llvm-config echo "Validating API structure..." ./tools/api/validate_structure.py + echo "Testing API and API Boosting..." + bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild @envoy_api_canonical//test/... @envoy_api_canonical//tools/... \ + @envoy_api_canonical//tools:tap2pcap_test @envoy_dev//clang_tools/api_booster/... echo "Building API..." bazel build "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild @envoy_api_canonical//envoy/... - echo "Testing API..." - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild @envoy_api_canonical//test/... @envoy_api_canonical//tools/... \ - @envoy_api_canonical//tools:tap2pcap_test - echo "Testing API boosting (unit tests)..." - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild @envoy_dev//clang_tools/api_booster/... echo "Testing API boosting (golden C++ tests)..." # We use custom BAZEL_BUILD_OPTIONS here; the API booster isn't capable of working with libc++ yet. - LLVM_CONFIG="${LLVM_ROOT}"/bin/llvm-config BAZEL_BUILD_OPTIONS="--config=clang" python3.8 ./tools/api_boost/api_boost_test.py + BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS[*]}" python3.8 ./tools/api_boost/api_boost_test.py exit 0 elif [[ "$CI_TARGET" == "bazel.coverage" || "$CI_TARGET" == "bazel.fuzz_coverage" ]]; then setup_clang_toolchain diff --git a/ci/go_mirror.sh b/ci/go_mirror.sh index 80be4cc0b5321..00e65816eb604 100755 --- a/ci/go_mirror.sh +++ b/ci/go_mirror.sh @@ -2,7 +2,11 @@ set -e -if [ -z "$CIRCLE_PULL_REQUEST" ] && [ "$CIRCLE_BRANCH" == "master" ] -then - tools/api/generate_go_protobuf.py +MAIN_BRANCH="refs/heads/azp_api" + +# shellcheck source=ci/setup_cache.sh +. "$(dirname "$0")"/setup_cache.sh + +if [[ "${AZP_BRANCH}" == "${MAIN_BRANCH}" ]]; then + BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_EXTRA_OPTIONS}" tools/api/generate_go_protobuf.py fi diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index 003a5ab28b76a..842b51b6ce894 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -56,6 +56,7 @@ mkdir -p "${ENVOY_DOCKER_BUILD_DIR}" [[ -t 1 ]] && ENVOY_DOCKER_OPTIONS+=("-it") [[ -f .git ]] && [[ ! -d .git ]] && ENVOY_DOCKER_OPTIONS+=(-v "$(git rev-parse --git-common-dir):$(git rev-parse --git-common-dir)") +[[ -n "${SSH_AUTH_SOCK}" ]] && ENVOY_DOCKER_OPTIONS+=(-v "${SSH_AUTH_SOCK}:${SSH_AUTH_SOCK}" -e SSH_AUTH_SOCK) export ENVOY_BUILD_IMAGE="${IMAGE_NAME}:${IMAGE_ID}" diff --git a/tools/api/generate_go_protobuf.py b/tools/api/generate_go_protobuf.py index 0cd15b6374493..c8867fef7515d 100755 --- a/tools/api/generate_go_protobuf.py +++ b/tools/api/generate_go_protobuf.py @@ -4,17 +4,21 @@ from subprocess import check_call import glob import os +import shlex import shutil import sys import re +# Needed for CI to pass down bazel options. +BAZEL_BUILD_OPTIONS = shlex.split(os.environ.get('BAZEL_BUILD_OPTIONS', '')) + TARGETS = '@envoy_api//...' IMPORT_BASE = 'github.com/envoyproxy/go-control-plane' OUTPUT_BASE = 'build_go' REPO_BASE = 'go-control-plane' -BRANCH = 'master' +BRANCH = 'master-ci-test' MIRROR_MSG = 'Mirrored from envoyproxy/envoy @ ' -USER_NAME = 'go-control-plane(CircleCI)' +USER_NAME = 'go-control-plane(Azure Pipelines)' USER_EMAIL = 'go-control-plane@users.noreply.github.com' @@ -32,7 +36,7 @@ def generateProtobufs(output): check_call([ 'bazel', 'build', '-c', 'fastbuild', '--experimental_proto_descriptor_sets_include_source_info' - ] + go_protos) + ] + BAZEL_BUILD_OPTIONS + go_protos) for rule in go_protos: # Example rule: @@ -67,9 +71,7 @@ def git(repo, *args): def cloneGoProtobufs(repo): # Create a local clone of go-control-plane - git(None, 'clone', 'git@github.com:envoyproxy/go-control-plane', repo) - git(repo, 'fetch') - git(repo, 'checkout', '-B', BRANCH, 'origin/master') + git(None, 'clone', 'git@github.com:envoyproxy/go-control-plane', repo, '-b', BRANCH) def findLastSyncSHA(repo): diff --git a/tools/api_boost/api_boost.py b/tools/api_boost/api_boost.py index eda6eaf940881..5cd9846bcf218 100755 --- a/tools/api_boost/api_boost.py +++ b/tools/api_boost/api_boost.py @@ -132,7 +132,6 @@ def ApiBoostTree(target_paths, sp.run([ 'bazel', 'build', - '--config=libc++', '--strip=always', ] + BAZEL_BUILD_OPTIONS + dep_lib_build_targets, check=True) From 0b5bceb856af0e9e912d3032194fc09cb3d6ad2f Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Tue, 13 Oct 2020 15:38:45 -0700 Subject: [PATCH 2/3] use main branch Signed-off-by: Lizan Zhou --- .azure-pipelines/pipelines.yml | 6 ++--- .circleci/config.yml | 43 ------------------------------- ci/api_mirror.sh | 4 +-- ci/go_mirror.sh | 2 +- tools/api/generate_go_protobuf.py | 2 +- 5 files changed, 6 insertions(+), 51 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml index 0b46ca296a17f..f8c5ce9721740 100644 --- a/.azure-pipelines/pipelines.yml +++ b/.azure-pipelines/pipelines.yml @@ -138,6 +138,7 @@ jobs: - job: api displayName: "data-plane-api sync" dependsOn: [] + condition: and(succeeded(), eq(variables['PostSubmit'], true)) pool: vmImage: "ubuntu-18.04" steps: @@ -147,18 +148,17 @@ jobs: sshPublicKey: "$(DataPlaneApiPublicKey)" sshPassphrase: "$(SshDeployKeyPassphrase)" sshKeySecureFile: "$(DataPlaneApiPrivateKey)" - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true)) - bash: ci/api_mirror.sh displayName: "Sync data-plane-api" workingDirectory: $(Build.SourcesDirectory) env: AZP_BRANCH: $(Build.SourceBranch) - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true)) - job: go_control_plane displayName: "go-control-plane sync" dependsOn: [] + condition: and(succeeded(), eq(variables['PostSubmit'], true)) steps: - task: InstallSSHKey@0 inputs: @@ -166,7 +166,6 @@ jobs: sshPublicKey: "$(GoControlPlanePublicKey)" sshPassphrase: "$(SshDeployKeyPassphrase)" sshKeySecureFile: "$(GoControlPlanePrivateKey)" - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true)) - bash: | cp -a ~/.ssh $(Build.StagingDirectory)/ @@ -179,7 +178,6 @@ jobs: BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) AZP_BRANCH: $(Build.SourceBranch) - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true)) - job: bazel displayName: "Linux-x64" diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index e04a9737d8f91..0000000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,43 +0,0 @@ -version: 2.1 - -executors: - ubuntu-build: - description: "A regular build executor based on ubuntu image" - docker: - # NOTE: Update this from https://github.com/envoyproxy/envoy-build-tools/blob/master/toolchains/rbe_toolchains_config.bzl#L8 - - image: envoyproxy/envoy-build-ubuntu:b480535e8423b5fd7c102fd30c92f4785519e33a - resource_class: xlarge - working_directory: /source - -jobs: - api: - executor: ubuntu-build - steps: - - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken - - checkout - - run: ci/do_circle_ci.sh bazel.api - - add_ssh_keys: - fingerprints: - - "fb:f3:fe:be:1c:b2:ec:b6:25:f9:7b:a6:87:54:02:8c" - - run: ci/api_mirror.sh - - store_artifacts: - path: /build/envoy/generated - destination: / - - go_control_plane_mirror: - executor: ubuntu-build - steps: - - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken - - checkout - - run: ci/do_circle_ci.sh bazel.api - - add_ssh_keys: - fingerprints: - - "9d:3b:fe:7c:09:3b:ce:a9:6a:de:de:41:fb:6b:52:62" - - run: ci/go_mirror.sh - -workflows: - version: 2 - all: - jobs: - - api - - go_control_plane_mirror diff --git a/ci/api_mirror.sh b/ci/api_mirror.sh index 09868f72f1d30..03e8ab85d80cb 100755 --- a/ci/api_mirror.sh +++ b/ci/api_mirror.sh @@ -3,8 +3,8 @@ set -e CHECKOUT_DIR=../data-plane-api -MAIN_BRANCH="refs/heads/azp_api" -API_MAIN_BRANCH="master-ci-test" +MAIN_BRANCH="refs/heads/master" +API_MAIN_BRANCH="master" if [[ "${AZP_BRANCH}" == "${MAIN_BRANCH}" ]]; then echo "Cloning..." diff --git a/ci/go_mirror.sh b/ci/go_mirror.sh index 00e65816eb604..63f96d0d79697 100755 --- a/ci/go_mirror.sh +++ b/ci/go_mirror.sh @@ -2,7 +2,7 @@ set -e -MAIN_BRANCH="refs/heads/azp_api" +MAIN_BRANCH="refs/heads/master" # shellcheck source=ci/setup_cache.sh . "$(dirname "$0")"/setup_cache.sh diff --git a/tools/api/generate_go_protobuf.py b/tools/api/generate_go_protobuf.py index c8867fef7515d..5b25de2dbb0a4 100755 --- a/tools/api/generate_go_protobuf.py +++ b/tools/api/generate_go_protobuf.py @@ -16,7 +16,7 @@ IMPORT_BASE = 'github.com/envoyproxy/go-control-plane' OUTPUT_BASE = 'build_go' REPO_BASE = 'go-control-plane' -BRANCH = 'master-ci-test' +BRANCH = 'master' MIRROR_MSG = 'Mirrored from envoyproxy/envoy @ ' USER_NAME = 'go-control-plane(Azure Pipelines)' USER_EMAIL = 'go-control-plane@users.noreply.github.com' From b459461aec995dd145afe780643565310c979215 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Tue, 13 Oct 2020 21:03:35 -0700 Subject: [PATCH 3/3] comment Signed-off-by: Lizan Zhou --- ci/do_ci.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 3e9eb939ae845..f3958aeaedf6d 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -312,7 +312,8 @@ elif [[ "$CI_TARGET" == "bazel.compile_time_options" ]]; then collect_build_profile build exit 0 elif [[ "$CI_TARGET" == "bazel.api" ]]; then - # Use libstdc++ because the API booster isn't capable of working with libc++ yet. + # Use libstdc++ because the API booster links to prebuilt libclang*/libLLVM* installed in /opt/llvm/lib, + # which is built with libstdc++. Using libstdc++ for whole of the API CI job to avoid unnecessary rebuild. ENVOY_STDLIB="libstdc++" setup_clang_toolchain export LLVM_CONFIG="${LLVM_ROOT}"/bin/llvm-config