diff --git a/.circleci/config.yml b/.circleci/config.yml index 2ade7ad40..f82b4cc15 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,6 @@ references: - envoy-build-image: &envoy-build-image # November 10th, 2020 - envoyproxy/envoy-build-ubuntu:19a268cfe3d12625380e7c61d2467c8779b58b56 + envoy-build-image: &envoy-build-image # February 14th, 2021 + envoyproxy/envoy-build-ubuntu:c8fa4235714003ba0896287ee2f91cae06e0e407 version: 2 jobs: build: diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 1ba6beb17..b4f8a5c40 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -56,15 +56,19 @@ important maintenance task. When performing the update, follow this procedure: ``` INFO: SHA256 (https://github.com/envoyproxy/envoy/archive/9753819331d1547c4b8294546a6461a3777958f5.tar.gz) = f4d26c7e78c0a478d959ea8bc877f260d4658a8b44e294e3a400f20ad44d41a3 ``` - 1. Update `ENVOY_SHA` in [bazel/repositories.bzl](bazel/repositories.bzl) to +1. Update `ENVOY_SHA` in [bazel/repositories.bzl](bazel/repositories.bzl) to this value. 1. Sync (copy) [.bazelrc](.bazelrc) from [Envoy's version](https://github.com/envoyproxy/envoy/blob/main/.bazelrc) to update our build configurations. Be sure to retain our local modifications, all lines that are unique to Nighthawk are marked with comment `# unique`. +1. In the updated [.bazelrc](.bazelrc) search for `experimental_docker_image`. + Copy the SHA and update `envoy-build-image:sha` over at the top of [.circleci/config.yml](.circleci/config.yml). 1. Sync (copy) [.bazelversion](.bazelversion) from [Envoy's version](https://github.com/envoyproxy/envoy/blob/main/.bazelversion) to ensure we are using the same build system version. +1. Sync (copy) [ci/run_envoy_docker.sh](ci/run_envoy_docker.sh) from + [Envoy's version](https://github.com/envoyproxy/envoy/blob/main/ci/run_envoy_docker.sh). 1. Run `ci/do_ci.sh test`. Sometimes the dependency update comes with changes that break our build. Include any changes required to Nighthawk to fix that in the same PR. diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 792f617a3..0fe4ac51a 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -34,8 +34,9 @@ function do_test() { } function do_clang_tidy() { - # TODO(#546): deflake clang tidy runs, and remove '|| true' here. - ci/run_clang_tidy.sh || true + # clang-tidy will warn on standard library issues with libc++ + BAZEL_BUILD_OPTIONS=("--config=clang" "${BAZEL_BUILD_OPTIONS[@]}") + BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS[*]}" NUM_CPUS=4 ci/run_clang_tidy.sh } function do_unit_test_coverage() { diff --git a/ci/run_clang_tidy.sh b/ci/run_clang_tidy.sh index 040b5a46b..6412bedc1 100755 --- a/ci/run_clang_tidy.sh +++ b/ci/run_clang_tidy.sh @@ -31,7 +31,7 @@ echo "Generating compilation database..." # Do not run clang-tidy against win32 impl # TODO(scw00): We should run clang-tidy against win32 impl once we have clang-cl support for Windows function exclude_win32_impl() { - grep -v source/common/filesystem/win32/ | grep -v source/common/common/win32 | grep -v source/exe/win32 | grep -v source/common/api/win32 + grep -v source/common/filesystem/win32/ | grep -v source/common/common/win32 | grep -v source/exe/win32 | grep -v source/common/api/win32 | grep -v source/common/event/win32 } # Do not run clang-tidy against macOS impl @@ -81,8 +81,13 @@ function exclude_wasm_test_data() { grep -v wasm/test_data } +# Exclude files which are part of the Wasm examples +function exclude_wasm_examples() { + grep -v examples/wasm +} + function filter_excludes() { - exclude_check_format_testdata | exclude_headersplit_testdata | exclude_chromium_url | exclude_win32_impl | exclude_macos_impl | exclude_third_party | exclude_wasm_emscripten | exclude_wasm_sdk | exclude_wasm_host | exclude_wasm_test_data + exclude_check_format_testdata | exclude_headersplit_testdata | exclude_chromium_url | exclude_win32_impl | exclude_macos_impl | exclude_third_party | exclude_wasm_emscripten | exclude_wasm_sdk | exclude_wasm_host | exclude_wasm_test_data | exclude_wasm_examples } function run_clang_tidy() { @@ -108,9 +113,7 @@ elif [[ "${RUN_FULL_CLANG_TIDY}" == 1 ]]; then run_clang_tidy else if [[ -z "${DIFF_REF}" ]]; then - if [[ "${BUILD_REASON}" == "PullRequest" ]]; then - DIFF_REF="remotes/origin/${SYSTEM_PULLREQUEST_TARGETBRANCH}" - elif [[ "${BUILD_REASON}" == *CI ]]; then + if [[ "${BUILD_REASON}" == *CI ]]; then DIFF_REF="HEAD^" else DIFF_REF=$("${ENVOY_SRCDIR}"/tools/git/last_github_commit.sh) diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index c6b91fae5..0ba9c5ec9 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -2,26 +2,99 @@ set -e -. ci/envoy_build_sha.sh +# shellcheck source=ci/envoy_build_sha.sh +. "$(dirname "$0")"/envoy_build_sha.sh -# We run as root and later drop permissions. This is required to setup the USER -# in useradd below, which is need for correct Python execution in the Docker -# environment. -USER=root -USER_GROUP=root +function is_windows() { + [[ "$(uname -s)" == *NT* ]] +} + +read -ra ENVOY_DOCKER_OPTIONS <<< "${ENVOY_DOCKER_OPTIONS:-}" + +# TODO(phlax): uppercase these env vars +export HTTP_PROXY="${http_proxy:-}" +export HTTPS_PROXY="${https_proxy:-}" +export NO_PROXY="${no_proxy:-}" + +if is_windows; then + [[ -z "${IMAGE_NAME}" ]] && IMAGE_NAME="envoyproxy/envoy-build-windows2019" + # TODO(sunjayBhatia): Currently ENVOY_DOCKER_OPTIONS is ignored on Windows because + # CI sets it to a Linux-specific value. Undo this once https://github.com/envoyproxy/envoy/issues/13272 + # is resolved. + ENVOY_DOCKER_OPTIONS=() + DEFAULT_ENVOY_DOCKER_BUILD_DIR=C:/Windows/Temp/envoy-docker-build + BUILD_DIR_MOUNT_DEST=C:/build + # Replace MSYS style drive letter (/c/) with driver letter designation (C:/) + SOURCE_DIR=$(echo "${PWD}" | sed -E "s#/([a-zA-Z])/#\1:/#") + SOURCE_DIR_MOUNT_DEST=C:/source + START_COMMAND=("bash" "-c" "cd source && $*") +else + [[ -z "${IMAGE_NAME}" ]] && IMAGE_NAME="envoyproxy/envoy-build-ubuntu" + # We run as root and later drop permissions. This is required to setup the USER + # in useradd below, which is need for correct Python execution in the Docker + # environment. + ENVOY_DOCKER_OPTIONS+=(-u root:root) + ENVOY_DOCKER_OPTIONS+=(-v /var/run/docker.sock:/var/run/docker.sock) + ENVOY_DOCKER_OPTIONS+=(--cap-add SYS_PTRACE --cap-add NET_RAW --cap-add NET_ADMIN) + DEFAULT_ENVOY_DOCKER_BUILD_DIR=/tmp/envoy-docker-build + BUILD_DIR_MOUNT_DEST=/build + SOURCE_DIR="${PWD}" + SOURCE_DIR_MOUNT_DEST=/source + START_COMMAND=("/bin/bash" "-lc" "groupadd --gid $(id -g) -f envoygroup \ + && useradd -o --uid $(id -u) --gid $(id -g) --no-create-home --home-dir /build envoybuild \ + && usermod -a -G pcap envoybuild \ + && chown envoybuild:envoygroup /build \ + && sudo -EHs -u envoybuild bash -c 'cd /source && $*'") +fi -[[ -z "${IMAGE_NAME}" ]] && IMAGE_NAME="envoyproxy/envoy-build-ubuntu" # The IMAGE_ID defaults to the CI hash but can be set to an arbitrary image ID (found with 'docker # images'). [[ -z "${IMAGE_ID}" ]] && IMAGE_ID="${ENVOY_BUILD_SHA}" -[[ -z "${ENVOY_DOCKER_BUILD_DIR}" ]] && ENVOY_DOCKER_BUILD_DIR=/tmp/envoy-docker-build +[[ -z "${ENVOY_DOCKER_BUILD_DIR}" ]] && ENVOY_DOCKER_BUILD_DIR="${DEFAULT_ENVOY_DOCKER_BUILD_DIR}" +# Replace backslash with forward slash for Windows style paths +ENVOY_DOCKER_BUILD_DIR="${ENVOY_DOCKER_BUILD_DIR//\\//}" +mkdir -p "${ENVOY_DOCKER_BUILD_DIR}" -[[ -f .git ]] && [[ ! -d .git ]] && GIT_VOLUME_OPTION="-v $(git rev-parse --git-common-dir):$(git rev-parse --git-common-dir)" +[[ -t 1 ]] && ENVOY_DOCKER_OPTIONS+=("-it") +[[ -f .git ]] && [[ ! -d .git ]] && ENVOY_DOCKER_OPTIONS+=(-v "$(git rev-parse --git-common-dir):$(git rev-parse --git-common-dir)") +[[ -n "${SSH_AUTH_SOCK}" ]] && ENVOY_DOCKER_OPTIONS+=(-v "${SSH_AUTH_SOCK}:${SSH_AUTH_SOCK}" -e SSH_AUTH_SOCK) + +export ENVOY_BUILD_IMAGE="${IMAGE_NAME}:${IMAGE_ID}" -mkdir -p "${ENVOY_DOCKER_BUILD_DIR}" # Since we specify an explicit hash, docker-run will pull from the remote repo if missing. -docker run --rm -t -i -e HTTP_PROXY=${http_proxy} -e HTTPS_PROXY=${https_proxy} \ - -u "${USER}":"${USER_GROUP}" -v "${ENVOY_DOCKER_BUILD_DIR}":/build ${GIT_VOLUME_OPTION} \ - -v "$PWD":/source -e NUM_CPUS --cap-add SYS_PTRACE --cap-add NET_RAW --cap-add NET_ADMIN "${IMAGE_NAME}":"${IMAGE_ID}" \ - /bin/bash -lc "groupadd --gid $(id -g) -f envoygroup && useradd -o --uid $(id -u) --gid $(id -g) --no-create-home \ - --home-dir /source envoybuild && usermod -a -G pcap envoybuild && su envoybuild -c \"cd source && $*\"" +docker run --rm \ + "${ENVOY_DOCKER_OPTIONS[@]}" \ + -v "${ENVOY_DOCKER_BUILD_DIR}":"${BUILD_DIR_MOUNT_DEST}" \ + -v "${SOURCE_DIR}":"${SOURCE_DIR_MOUNT_DEST}" \ + -e AZP_BRANCH \ + -e HTTP_PROXY \ + -e HTTPS_PROXY \ + -e NO_PROXY \ + -e BAZEL_STARTUP_OPTIONS \ + -e BAZEL_BUILD_EXTRA_OPTIONS \ + -e BAZEL_EXTRA_TEST_OPTIONS \ + -e BAZEL_REMOTE_CACHE \ + -e ENVOY_STDLIB \ + -e BUILD_REASON \ + -e BAZEL_REMOTE_INSTANCE \ + -e GCP_SERVICE_ACCOUNT_KEY \ + -e NUM_CPUS \ + -e ENVOY_RBE \ + -e ENVOY_BUILD_IMAGE \ + -e ENVOY_SRCDIR \ + -e ENVOY_BUILD_TARGET \ + -e SYSTEM_PULLREQUEST_PULLREQUESTNUMBER \ + -e GCS_ARTIFACT_BUCKET \ + -e GITHUB_TOKEN \ + -e BUILD_SOURCEBRANCHNAME \ + -e BAZELISK_BASE_URL \ + -e ENVOY_BUILD_ARCH \ + -e SLACK_TOKEN \ + -e BUILD_URI\ + -e REPO_URI \ + -e SYSTEM_STAGEDISPLAYNAME \ + -e SYSTEM_JOBDISPLAYNAME \ + -e SYSTEM_PULLREQUEST_PULLREQUESTID \ + "${ENVOY_BUILD_IMAGE}" \ + "${START_COMMAND[@]}" + \ No newline at end of file