diff --git a/.travis.yml b/.travis.yml index 7269cbb3050f0..7e625bae01f07 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,11 +7,10 @@ install: matrix: fast_finish: true env: - - TEST_TYPE=debug + - TEST_TYPE=bazel.release + - TEST_TYPE=bazel.asan - TEST_TYPE=bazel.coverage - - TEST_TYPE=bazel.debug - - TEST_TYPE=normal - - TEST_TYPE=asan + - TEST_TYPE=check_format - TEST_TYPE=docs script: ./ci/ci_steps.sh diff --git a/Makefile b/Makefile deleted file mode 100644 index 15e00c92e9164..0000000000000 --- a/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -DOCS_OUTPUT_DIR ?= generated/docs - -.PHONY: docs -docs: - rm -fr generated/docs - mkdir -p generated/docs - docs/build.sh $(DOCS_OUTPUT_DIR) diff --git a/bazel/README.md b/bazel/README.md index 3fd89a7ca23c3..887f708bad273 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -178,6 +178,9 @@ report is available in `generated/coverage/coverage.html`. Envoy proper. To remove the artifacts for the external dependencies run `bazel clean --expunge`. +If something goes really wrong and none of the above work to resolve a stale build issue, you can +always remove your Bazel cache completely. It is likely located in `~/.cache/bazel`. + # Adding or maintaining Envoy build rules See the [developer guide for writing Envoy Bazel rules](DEVELOPER.md). diff --git a/bazel/cc_configure.bzl b/bazel/cc_configure.bzl index cd936ccf7b2b8..8a7a69c4f0b36 100644 --- a/bazel/cc_configure.bzl +++ b/bazel/cc_configure.bzl @@ -228,7 +228,7 @@ def _crosstool_content(repository_ctx, cc, cpu_value, darwin): "builtin_sysroot": "", "compiler": _get_env_var(repository_ctx, "BAZEL_COMPILER", "compiler", False), "host_system_name": _get_env_var(repository_ctx, "BAZEL_HOST_SYSTEM", "local", False), - "needsPic": True, + "needsPic": False, "supports_gold_linker": supports_gold_linker, "supports_incremental_linker": False, "supports_fission": False, diff --git a/bazel/envoy_build_system.bzl b/bazel/envoy_build_system.bzl index fac4bd4741338..a53bc9ebf8a8e 100644 --- a/bazel/envoy_build_system.bzl +++ b/bazel/envoy_build_system.bzl @@ -4,7 +4,7 @@ def envoy_package(): native.package(default_visibility = ["//visibility:public"]) # Compute the final copts based on various options. -def envoy_copts(repository): +def envoy_copts(repository, test = False): return [ # TODO(htuch): Remove this when Bazel bringup is done. "-DBAZEL_BRINGUP", @@ -17,7 +17,7 @@ def envoy_copts(repository): "-std=c++0x", ] + select({ # Bazel adds an implicit -DNDEBUG for opt. - repository + "//bazel:opt_build": ["-ggdb3"], + repository + "//bazel:opt_build": [] if test else ["-ggdb3"], repository + "//bazel:fastbuild_build": [], repository + "//bazel:dbg_build": ["-ggdb3"], }) + select({ @@ -166,7 +166,7 @@ def envoy_cc_test(name, ) native.cc_test( name = name, - copts = envoy_copts(repository), + copts = envoy_copts(repository, test = True), # TODO(mattklein123): It's not great that we universally link against the following libs. # In particular, -latomic is not needed on all platforms. Make this more granular. linkopts = ["-pthread", "-latomic"], @@ -195,7 +195,7 @@ def envoy_cc_test_library(name, srcs = srcs, hdrs = hdrs, data = data, - copts = envoy_copts(repository), + copts = envoy_copts(repository, test = True), testonly = 1, deps = deps + [envoy_external_dep_path(dep) for dep in external_deps] + [ envoy_external_dep_path('googletest'), @@ -263,6 +263,7 @@ def envoy_proto_library(name, srcs = [], deps = []): default_runtime = "//external:protobuf", protoc = "//external:protoc", deps = deps, + linkstatic = 1, ) # We can't use include_prefix directly in cc_proto_library, since it # confuses protoc. Instead, we create a shim cc_library that performs the diff --git a/ci/Dockerfile-envoy-image b/ci/Dockerfile-envoy-image index 78e8d6f7a2d8f..bd84854292677 100644 --- a/ci/Dockerfile-envoy-image +++ b/ci/Dockerfile-envoy-image @@ -1,6 +1,6 @@ FROM ubuntu:14.04 -ADD build_normal/source/exe/envoy /usr/local/bin/envoy +ADD build_release/envoy /usr/local/bin/envoy RUN apt-get update && apt-get install -y \ build-essential RUN strip /usr/local/bin/envoy diff --git a/ci/README.md b/ci/README.md index a58a5dae89282..2691f77de399f 100644 --- a/ci/README.md +++ b/ci/README.md @@ -13,71 +13,28 @@ binary built from the latest tip of master that passed tests. ## Alpine envoy image -Minimal images based on alpine Linux allow for quicker deployment of Envoy. Two alpine based images are built, +Minimal images based on Alpine Linux allow for quicker deployment of Envoy. Two Alpine based images are built, one with an Envoy binary with debug (`lyft/envoy-alpine-debug`) symbols and one stripped of them (`lyft/envoy-alpine`). Both images are pushed with two different tags: `` and `latest`. Parallel to the Ubuntu images above, `` corresponds to the master commit at which the binary was compiled, and `latest` corresponds to a binary built from the latest tip of master that passed tests. -# Building a debug image -An example basic invocation to build a debug image and run all tests is: +# Building and running tests as a developer -```bash -docker pull lyft/envoy-build:latest && docker run -t -i -u $(id -u):$(id -g) -v :/build -v :/source lyft/envoy-build:latest /bin/bash -c "cd /source && ci/do_ci.sh debug" -``` - -On OSX using the command below may work better. Unlike on Linux, users are not -synced between the container and OS. Additionally, the bind mount will not -create artifacts with the same ownership as in the container ([read more about -osxfs][osxfs]). - -```bash -docker pull lyft/envoy-build:latest && docker run -t -i -u root:root -v :/build -v :/source lyft/envoy-build:latest /bin/bash -c "cd /source && ci/do_ci.sh debug" -``` - -This bind mounts ``, which allows for changes on the local -filesystem to be consumed and outputs build artifacts in `/build`. -The static Envoy binary can be found in `/build_debug/source/exe/envoy`. - -The `do_ci.sh` targets are: - -* `asan` — build and run tests with [AddressSanitizer](https://github.com/google/sanitizers/wiki/AddressSanitizer). -* `coverage` — build and run tests, generating coverage information in `/build_coverage/coverage.html`. -* `debug` — build debug binary and run tests. -* `bazel.coverage` — build and run tests with Bazel, generating coverage information in `/generated/coverage/coverage.html`. -* `bazel.debug` — build debug binary and run tests with Bazel. -* `fix_format`— run `clang-format` 3.6 on entire source tree. -* `normal` — build unstripped optimized binary and run tests . -* `server_only` — build stripped optimized binary only. - -A convenient shell function to define is: +An example basic invocation to build the Envoy static binary (using the Bazel `fastbuild` type) is: ```bash -run_envoy_docker() { docker pull lyft/envoy-build:latest && docker run -t -i -u $(id -u):$(id -g) -v /tmp/envoy-docker-build:/build -v $PWD:/source lyft/envoy-build:latest /bin/bash -c "cd /source && $*";} +./ci/run_envoy_docker.sh './ci/do_ci.sh bazel.fastbuild' ``` -Or on OSX. - -```bash -run_envoy_docker() { docker pull lyft/envoy-build:latest && docker run -t -i -u root:root -v /tmp/envoy-docker-build:/build -v $PWD:/source lyft/envoy-build:latest /bin/bash -c "cd /source && $*";} -``` - -This then allows for a simple invocation of `run_envoy_docker './ci/do_ci.sh debug'` from the -Envoy source tree. - -## Advanced developer features - -* Any parameters following the `do_ci.sh` target are passed in as command-line - arguments to the `envoy-test` binary during unit test execution. This allows - for [GTest](https://github.com/google/googletest) flags to be passed, e.g. - `run_envoy_docker './ci/do_ci.sh debug --gtest_filter="*Dns*"'`. - -* A `UNIT_TEST_ONLY` environment variable is available to control test execution to limit testing to - just unit tests, e.g. `run_envoy_docker 'UNIT_TEST_ONLY=1 ./ci/do_ci.sh debug --gtest_filter="*Dns*"'`. - -* A `RUN_TEST_UNDER` environment variable is available to specify an executable to run the tests - under. For example, to run a subset of tests under `gdb`: `run_envoy_docker 'RUN_TEST_UNDER="gdb --args" UNIT_TEST_ONLY=1 ./ci/do_ci.sh debug --gtest_filter="*Dns*"'`. - -* A `SKIP_CHECK_FORMAT` environment variable is available to skip `clang-format` checks while developing locally, e.g. `run_envoy_docker 'SKIP_CHECK_FORMAT=1 ./ci/do_ci.sh debug'`. +The output artifacts can be found in `/build/envoy/bazel-bin/source/exe/envoy-static` in the +container. +The `./ci/run_envoy_docker.sh './ci/do_ci.sh '` targets are: -[osxfs]: https://docs.docker.com/docker-for-mac/osxfs/ +* `bazel.asan` — build and run tests under `-c dbg --config=asan`. +* `bazel.fastbuild` — build Envoy static binary under `-c fastbuild`. +* `bazel.fastbuild.test` — build and run tests under `-c fastbuild`. +* `bazel.release` — build Envoy static binary and run tests under `-c opt`. +* `bazel.coverage` — build and run tests under `-c dbg`, generating coverage information in `/generated/coverage/coverage.html`. +* `check_format`— run `clang-format` 3.6 and `buildifier` on entire source tree. +* `fix_format`— run and enforce `clang-format` 3.6 and `buildifier` on entire source tree. diff --git a/ci/build_alpine_container/Makefile b/ci/build_alpine_container/Makefile index e98fba14d4662..828d1d334c477 100644 --- a/ci/build_alpine_container/Makefile +++ b/ci/build_alpine_container/Makefile @@ -1,7 +1,8 @@ PHONY: all -ENVOY_TAG?="latest" -ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) +ENVOY_TAG ?= "latest" +ENVOY_BUILD_DIR ?= "/tmp/envoy-docker-build" +ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) all: binary container test @@ -11,9 +12,9 @@ binary: mkdir -p ./bin/stripped mkdir -p ./bin/debug mkdir -p ./bin/configs - docker run -t -i --name build-envoy-alpine-ct -v ${ROOT_DIR}/../../:/source lyft/envoy-build:latest /bin/bash -c \ - "set -x && cd /source && /source/ci/do_ci.sh server_only && \ - cp /source/build_normal/source/exe/envoy /tmp/ && \ + docker run -t -i --name build-envoy-alpine-ct -v ${ROOT_DIR}/../../:/source -v $(ENVOY_BUILD_DIR):/build lyft/envoy-build:latest /bin/bash -c \ + "set -x && cd /source && + cp /build/envoy/source/exe/envoy /tmp/envoy && \ cd /tmp/ && strip envoy && \ mv /tmp/envoy /source/ci/build_alpine_container/bin/stripped/ && \ cp /source/build_normal/source/exe/envoy /source/ci/build_alpine_container/bin/debug/ && \ diff --git a/ci/build_alpine_container/README.md b/ci/build_alpine_container/README.md deleted file mode 100644 index 18b38bf5dd6f0..0000000000000 --- a/ci/build_alpine_container/README.md +++ /dev/null @@ -1,17 +0,0 @@ -This directory contains a Dockerfile that builds an envoy image -on alpine as the base instead of the default ubuntu. - -Invoking `make` will build the binary followed by the container. - -Invoke `make binary` to build the envoy binary. This will take a while, -when invoked the first time, around 20-30 mins on a single socket -2.9 GHz + 16 GB 2GHz memory i7 system. It's recommended to build it on a -beefy machine for a quicker build. - -Invoke `make container` to build the base alpine based Envoy container. -It will contain the envoy binary under /usr/local/bin/. - -Invoke `make clean` to clean up the built bin directory. - -The base alpine image is frolvlad/alpine-glibc, which has glibc -built into it. diff --git a/ci/build_amazon_linux/build_amazon_linux.sh b/ci/build_amazon_linux/build_amazon_linux.sh deleted file mode 100755 index ccd71d0a4ae2f..0000000000000 --- a/ci/build_amazon_linux/build_amazon_linux.sh +++ /dev/null @@ -1,169 +0,0 @@ -# Build envoy binary on amazon linux ami. -# -# Much of this comes from: -# https://github.com/lyft/envoy/blob/master/ci/build_container/build_container.sh -# https://github.com/lyft/envoy/blob/master/ci/do_ci.sh -# -# Helpful links: -# https://lyft.github.io/envoy/docs/install/requirements.html -# https://lyft.github.io/envoy/docs/install/building.html -# https://gcc.gnu.org/wiki/InstallingGCC - -# prerequisite -# run in screen! -set -x -NUM_CPUS=`grep -c ^processor /proc/cpuinfo` -echo "building using $NUM_CPUS CPUs" -ENVOY_ROOT=/mnt/envoy-root -sudo yum install -y cmake gmp-devel mpfr-devel libmpc-devel libgcc48.i686 glibc-devel.i686 golang clang -sudo mkdir -p $ENVOY_ROOT -sudo chown `whoami`:`whoami` $ENVOY_ROOT -cd $ENVOY_ROOT - -# gcc -wget http://www.netgull.com/gcc/releases/gcc-4.9.4/gcc-4.9.4.tar.gz -tar zxf gcc-4.9.4.tar.gz -mkdir -p objdir -cd objdir -$PWD/../gcc-4.9.4/configure --prefix=$PWD/../gcc-4.9.4 --enable-languages=c,c++ -make -j $NUM_CPUS -make install -cd .. -export PATH=$PWD/gcc-4.9.4/bin:$PATH -export CC=`which gcc` -export CXX=`which g++` - -# Build artifacts -THIRDPARTY_DIR=$PWD/thirdparty -THIRDPARTY_BUILD=$PWD/thirdparty_build -mkdir -p $THIRDPARTY_DIR -mkdir -p $THIRDPARTY_BUILD -cd $THIRDPARTY_DIR - -# libevent -wget https://github.com/libevent/libevent/releases/download/release-2.0.22-stable/libevent-2.0.22-stable.tar.gz -tar xf libevent-2.0.22-stable.tar.gz -cd libevent-2.0.22-stable -./configure --prefix=$THIRDPARTY_BUILD --enable-shared=no --disable-libevent-regress --disable-openssl -make install -cd .. - -# boring ssl -git clone https://boringssl.googlesource.com/boringssl -cd boringssl -git reset --hard be2ee342d3781ddb954f91f8a7e660c6f59e87e5 -cmake . -make -cp -r include/* $THIRDPARTY_BUILD/include -cp ssl/libssl.a $THIRDPARTY_BUILD/lib -cp crypto/libcrypto.a $THIRDPARTY_BUILD/lib -cd .. - -# gperftools -wget https://github.com/gperftools/gperftools/releases/download/gperftools-2.5/gperftools-2.5.tar.gz -tar xf gperftools-2.5.tar.gz -cd gperftools-2.5 -./configure --prefix=$THIRDPARTY_BUILD --enable-shared=no --enable-frame-pointers -make install -cd .. - -# nghttp2 -wget https://github.com/nghttp2/nghttp2/releases/download/v1.14.1/nghttp2-1.14.1.tar.gz -tar xf nghttp2-1.14.1.tar.gz -cd nghttp2-1.14.1 -./configure --prefix=$THIRDPARTY_BUILD --enable-shared=no --enable-lib-only -make install -cd .. - -# protobuf -wget https://github.com/google/protobuf/releases/download/v3.0.0/protobuf-cpp-3.0.0.tar.gz -tar xf protobuf-cpp-3.0.0.tar.gz -cd protobuf-3.0.0 -./configure --prefix=$THIRDPARTY_BUILD --enable-shared=no -make install -cd .. - -# cotire -wget https://github.com/sakra/cotire/archive/cotire-1.7.8.tar.gz -tar xf cotire-1.7.8.tar.gz - -# spdlog -wget https://github.com/gabime/spdlog/archive/v0.11.0.tar.gz -tar xf v0.11.0.tar.gz - -# http-parser -wget -O http-parser-v2.7.0.tar.gz https://github.com/nodejs/http-parser/archive/v2.7.0.tar.gz -tar xf http-parser-v2.7.0.tar.gz -cd http-parser-2.7.0 -$CC -O2 -c http_parser.c -o http_parser.o -ar rcs libhttp_parser.a http_parser.o -cp libhttp_parser.a $THIRDPARTY_BUILD/lib -cp http_parser.h $THIRDPARTY_BUILD/include -cd .. - -# tclap -wget -O tclap-1.2.1.tar.gz https://sourceforge.net/projects/tclap/files/tclap-1.2.1.tar.gz/download -tar xf tclap-1.2.1.tar.gz - -# lightstep -wget https://github.com/lightstep/lightstep-tracer-cpp/releases/download/v0_19/lightstep-tracer-cpp-0.19.tar.gz -tar xf lightstep-tracer-cpp-0.19.tar.gz -cd lightstep-tracer-cpp-0.19 -./configure --disable-grpc --prefix=$THIRDPARTY_BUILD --enable-shared=no \ - protobuf_CFLAGS="-I$THIRDPARTY_BUILD/include" protobuf_LIBS="-L$THIRDPARTY_BUILD/lib -lprotobuf" -make install -cd .. - -# rapidjson -wget -O rapidjson-1.1.0.tar.gz https://github.com/miloyip/rapidjson/archive/v1.1.0.tar.gz -tar xf rapidjson-1.1.0.tar.gz - -# google test -wget -O googletest-1.8.0.tar.gz https://github.com/google/googletest/archive/release-1.8.0.tar.gz -tar xf googletest-1.8.0.tar.gz -cd googletest-release-1.8.0 -cmake -DCMAKE_INSTALL_PREFIX:PATH=$THIRDPARTY_BUILD . -make install -cd .. - -# gcovr -wget -O gcovr-3.3.tar.gz https://github.com/gcovr/gcovr/archive/3.3.tar.gz -tar xf gcovr-3.3.tar.gz - -# envoy -cd .. -export LD_LIBRARY_PATH=$PWD/gcc-4.9.4/lib64:$LD_LIBRARY_PATH -git clone https://github.com/lyft/envoy.git -mkdir -p envoy/build -cd envoy/build - -cmake \ --DENVOY_DEBUG:BOOL=OFF \ --DENVOY_STRIP:BOOL=ON \ --DCLANG-FORMAT:FILEPATH=clang-format \ --DCMAKE_CXX_FLAGS=-static-libstdc++ \ --DENVOY_COTIRE_MODULE_DIR:FILEPATH=$THIRDPARTY_DIR/cotire-cotire-1.7.8/CMake \ --DENVOY_GMOCK_INCLUDE_DIR:FILEPATH=$THIRDPARTY_BUILD/include \ --DENVOY_GPERFTOOLS_INCLUDE_DIR:FILEPATH=$THIRDPARTY_BUILD/include \ --DENVOY_GTEST_INCLUDE_DIR:FILEPATH=$THIRDPARTY_BUILD/include \ --DENVOY_HTTP_PARSER_INCLUDE_DIR:FILEPATH=$THIRDPARTY_BUILD/include \ --DENVOY_LIBEVENT_INCLUDE_DIR:FILEPATH=$THIRDPARTY_BUILD/include \ --DENVOY_NGHTTP2_INCLUDE_DIR:FILEPATH=$THIRDPARTY_BUILD/include \ --DENVOY_SPDLOG_INCLUDE_DIR:FILEPATH=$THIRDPARTY_DIR/spdlog-0.11.0/include \ --DENVOY_TCLAP_INCLUDE_DIR:FILEPATH=$THIRDPARTY_DIR/tclap-1.2.1/include \ --DENVOY_OPENSSL_INCLUDE_DIR:FILEPATH=$THIRDPARTY_BUILD/include \ --DENVOY_LIGHTSTEP_TRACER_INCLUDE_DIR:FILEPATH=$THIRDPARTY_BUILD/include \ --DENVOY_PROTOBUF_INCLUDE_DIR:FILEPATH=$THIRDPARTY_BUILD/include \ --DENVOY_PROTOBUF_PROTOC:FILEPATH=$THIRDPARTY_BUILD/bin/protoc \ --DENVOY_GCOVR:FILEPATH=$THIRDPARTY_DIR/gcovr-3.3/scripts/gcovr \ --DENVOY_RAPIDJSON_INCLUDE_DIR:FILEPATH=$THIRDPARTY_DIR/rapidjson-1.1.0/include \ --DENVOY_GCOVR_EXTRA_ARGS:STRING="-e test/* -e build/*" \ --DENVOY_EXE_EXTRA_LINKER_FLAGS:STRING=-L$THIRDPARTY_BUILD/lib \ --DENVOY_TEST_EXTRA_LINKER_FLAGS:STRING=-L$THIRDPARTY_BUILD/lib \ -.. - -cmake -L || /bin/true -make check_format -make -j $NUM_CPUS envoy -cd ../.. -ls -l envoy/build/source/exe/envoy diff --git a/ci/build_setup.sh b/ci/build_setup.sh index b586428a1bf8a..ee09fcacf30c7 100755 --- a/ci/build_setup.sh +++ b/ci/build_setup.sh @@ -1,7 +1,6 @@ #!/bin/bash -# Configure environment variables, generate makefiles and switch to build -# directory in preparation for an invocation of the generated build makefiles. +# Configure environment variables for Bazel build and test. set -e @@ -14,108 +13,68 @@ NUM_CPUS=`grep -c ^processor /proc/cpuinfo` export ENVOY_SRCDIR=/source -if [[ "$1" == bazel* ]] -then - # Create a fake home. Python site libs tries to do getpwuid(3) if we don't and the CI - # Docker image gets confused as it has no passwd entry when running non-root - # unless we do this. - FAKE_HOME=/tmp/fake_home - mkdir -p "${FAKE_HOME}" - export HOME="${FAKE_HOME}" - export PYTHONUSERBASE="${FAKE_HOME}" +# Create a fake home. Python site libs tries to do getpwuid(3) if we don't and the CI +# Docker image gets confused as it has no passwd entry when running non-root +# unless we do this. +FAKE_HOME=/tmp/fake_home +mkdir -p "${FAKE_HOME}" +export HOME="${FAKE_HOME}" +export PYTHONUSERBASE="${FAKE_HOME}" - export BUILD_DIR=/build - # Make sure that "docker run" has a -v bind mount for /build, since cmake - # users will only have a bind mount for /source. - if [[ ! -d "${BUILD_DIR}" ]] - then - echo "${BUILD_DIR} mount missing - did you forget -v :${BUILD_DIR}?" - exit 1 - fi - export ENVOY_CONSUMER_SRCDIR="${BUILD_DIR}/envoy-consumer" +export BUILD_DIR=/build +# Make sure that "docker run" has a -v bind mount for /build, since cmake +# users will only have a bind mount for /source. +if [[ ! -d "${BUILD_DIR}" ]] +then + echo "${BUILD_DIR} mount missing - did you forget -v :${BUILD_DIR}?" + exit 1 +fi +export ENVOY_CONSUMER_SRCDIR="${BUILD_DIR}/envoy-consumer" - # Make sure that /source doesn't contain /build on the underlying host - # filesystem, including via hard links or symlinks. We can get into weird - # loops with Bazel symlinking and gcovr's path traversal if this is true, so - # best to keep /source and /build in distinct directories on the host - # filesystem. - SENTINEL="${BUILD_DIR}"/bazel.sentinel - touch "${SENTINEL}" - if [[ -n "$(find -L "${ENVOY_SRCDIR}" -name "$(basename "${SENTINEL}")")" ]] - then - rm -f "${SENTINEL}" - echo "/source mount must not contain /build mount" - exit 1 - fi +# Make sure that /source doesn't contain /build on the underlying host +# filesystem, including via hard links or symlinks. We can get into weird +# loops with Bazel symlinking and gcovr's path traversal if this is true, so +# best to keep /source and /build in distinct directories on the host +# filesystem. +SENTINEL="${BUILD_DIR}"/bazel.sentinel +touch "${SENTINEL}" +if [[ -n "$(find -L "${ENVOY_SRCDIR}" -name "$(basename "${SENTINEL}")")" ]] +then rm -f "${SENTINEL}" + echo "/source mount must not contain /build mount" + exit 1 +fi +rm -f "${SENTINEL}" - # Environment setup. - export USER=bazel - export TEST_TMPDIR=/build/tmp - export BAZEL="bazel" - # Not sandboxing, since non-privileged Docker can't do nested namespaces. - BAZEL_OPTIONS="--package_path %workspace%:/source" - export BAZEL_QUERY_OPTIONS="${BAZEL_OPTIONS}" - export BAZEL_BUILD_OPTIONS="--strategy=Genrule=standalone --spawn_strategy=standalone \ - --verbose_failures ${BAZEL_OPTIONS} --action_env=HOME --action_env=PYTHONUSERBASE" - export BAZEL_TEST_OPTIONS="${BAZEL_BUILD_OPTIONS} --test_env=HOME --test_env=PYTHONUSERBASE" - [[ "${BAZEL_EXPUNGE}" == "1" ]] && "${BAZEL}" clean --expunge - ln -sf /thirdparty "${ENVOY_SRCDIR}"/ci/prebuilt - ln -sf /thirdparty_build "${ENVOY_SRCDIR}"/ci/prebuilt - - # Setup Envoy consuming project. - if [[ ! -a "${ENVOY_CONSUMER_SRCDIR}" ]] - then - # TODO(htuch): Update to non-htuch when https://github.com/lyft/envoy/issues/404 is sorted. - git clone https://github.com/htuch/envoy-consumer.git "${ENVOY_CONSUMER_SRCDIR}" - fi - cp -f "${ENVOY_SRCDIR}"/ci/WORKSPACE.consumer "${ENVOY_CONSUMER_SRCDIR}"/WORKSPACE - # This is the hash on https://github.com/htuch/envoy-consumer.git we pin to. - (cd "${ENVOY_CONSUMER_SRCDIR}" && git checkout 94e11fa753a1e787c82cccaec642eda5e5b61ed8) - - # Also setup some space for building Envoy standalone. - export ENVOY_BUILD_DIR="${BUILD_DIR}"/envoy - mkdir -p "${ENVOY_BUILD_DIR}" - cp -f "${ENVOY_SRCDIR}"/ci/WORKSPACE "${ENVOY_BUILD_DIR}" -else - # TODO(htuch): Remove everything below this comment when we turn off cmake. - if [[ "$1" == "coverage" ]]; then - EXTRA_CMAKE_FLAGS+=" -DENVOY_CODE_COVERAGE:BOOL=ON" - elif [[ "$1" == "asan" ]]; then - EXTRA_CMAKE_FLAGS+=" -DENVOY_SANITIZE:BOOL=ON -DENVOY_DEBUG:BOOL=OFF" - elif [[ "$1" == "debug" ]]; then - EXTRA_CMAKE_FLAGS+=" -DENVOY_DEBUG:BOOL=ON" - elif [[ "$1" == "server_only" ]]; then - EXTRA_CMAKE_FLAGS+=" -DENVOY_DEBUG:BOOL=OFF -DENVOY_STRIP:BOOL=ON" - else - EXTRA_CMAKE_FLAGS+=" -DENVOY_DEBUG:BOOL=OFF" - fi - - mkdir -p build_"$1" - cd build_"$1" - - cmake \ - ${EXTRA_CMAKE_FLAGS} \ - -DENVOY_COTIRE_MODULE_DIR:FILEPATH=/thirdparty/cotire-cotire-1.7.8/CMake \ - -DENVOY_GMOCK_INCLUDE_DIR:FILEPATH=/thirdparty_build/include \ - -DENVOY_GPERFTOOLS_INCLUDE_DIR:FILEPATH=/thirdparty_build/include \ - -DENVOY_GTEST_INCLUDE_DIR:FILEPATH=/thirdparty_build/include \ - -DENVOY_HTTP_PARSER_INCLUDE_DIR:FILEPATH=/thirdparty_build/include \ - -DENVOY_LIBEVENT_INCLUDE_DIR:FILEPATH=/thirdparty_build/include \ - -DENVOY_CARES_INCLUDE_DIR:FILEPATH=/thirdparty_build/include \ - -DENVOY_NGHTTP2_INCLUDE_DIR:FILEPATH=/thirdparty_build/include \ - -DENVOY_SPDLOG_INCLUDE_DIR:FILEPATH=/thirdparty/spdlog-0.11.0/include \ - -DENVOY_TCLAP_INCLUDE_DIR:FILEPATH=/thirdparty/tclap-1.2.1/include \ - -DENVOY_OPENSSL_INCLUDE_DIR:FILEPATH=/thirdparty_build/include \ - -DENVOY_LIGHTSTEP_TRACER_INCLUDE_DIR:FILEPATH=/thirdparty_build/include \ - -DENVOY_PROTOBUF_INCLUDE_DIR:FILEPATH=/thirdparty_build/include \ - -DENVOY_PROTOBUF_PROTOC:FILEPATH=/thirdparty_build/bin/protoc \ - -DENVOY_GCOVR:FILEPATH=/thirdparty/gcovr-3.3/scripts/gcovr \ - -DENVOY_RAPIDJSON_INCLUDE_DIR:FILEPATH=/thirdparty/rapidjson-1.1.0/include \ - -DENVOY_GCOVR_EXTRA_ARGS:STRING="-e test/* -e build/*" \ - -DENVOY_EXE_EXTRA_LINKER_FLAGS:STRING=-L/thirdparty_build/lib \ - -DENVOY_TEST_EXTRA_LINKER_FLAGS:STRING=-L/thirdparty_build/lib \ - .. +# Environment setup. +export USER=bazel +export TEST_TMPDIR=/build/tmp +export BAZEL="bazel" +# Not sandboxing, since non-privileged Docker can't do nested namespaces. +BAZEL_OPTIONS="--package_path %workspace%:/source" +export BAZEL_QUERY_OPTIONS="${BAZEL_OPTIONS}" +export BAZEL_BUILD_OPTIONS="--strategy=Genrule=standalone --spawn_strategy=standalone \ + --verbose_failures ${BAZEL_OPTIONS} --action_env=HOME --action_env=PYTHONUSERBASE \ + --jobs=${NUM_CPUS}" +export BAZEL_TEST_OPTIONS="${BAZEL_BUILD_OPTIONS} --test_env=HOME --test_env=PYTHONUSERBASE" +[[ "${BAZEL_EXPUNGE}" == "1" ]] && "${BAZEL}" clean --expunge +ln -sf /thirdparty "${ENVOY_SRCDIR}"/ci/prebuilt +ln -sf /thirdparty_build "${ENVOY_SRCDIR}"/ci/prebuilt - cmake -L || true +# Setup Envoy consuming project. +if [[ ! -a "${ENVOY_CONSUMER_SRCDIR}" ]] +then + # TODO(htuch): Update to non-htuch when https://github.com/lyft/envoy/issues/404 is sorted. + git clone https://github.com/htuch/envoy-consumer.git "${ENVOY_CONSUMER_SRCDIR}" fi +cp -f "${ENVOY_SRCDIR}"/ci/WORKSPACE.consumer "${ENVOY_CONSUMER_SRCDIR}"/WORKSPACE +mkdir -p "${ENVOY_CONSUMER_SRCDIR}"/tools +# Hack due to https://github.com/lyft/envoy/issues/838. +ln -sf "${ENVOY_SRCDIR}"/tools/bazel.rc "${ENVOY_CONSUMER_SRCDIR}"/tools/bazel.rc +# This is the hash on https://github.com/htuch/envoy-consumer.git we pin to. +(cd "${ENVOY_CONSUMER_SRCDIR}" && git checkout 94e11fa753a1e787c82cccaec642eda5e5b61ed8) + +# Also setup some space for building Envoy standalone. +export ENVOY_BUILD_DIR="${BUILD_DIR}"/envoy +mkdir -p "${ENVOY_BUILD_DIR}" +cp -f "${ENVOY_SRCDIR}"/ci/WORKSPACE "${ENVOY_BUILD_DIR}" diff --git a/ci/ci_steps.sh b/ci/ci_steps.sh index a06347ccef5f2..90788c7eaeb98 100755 --- a/ci/ci_steps.sh +++ b/ci/ci_steps.sh @@ -7,20 +7,27 @@ set -e # Lint travis file. travis lint .travis.yml --skip-completion-check -# Do a build matrix with different types of builds docs, coverage, normal, etc. -if [ $TEST_TYPE == "docs" ] +# Where the Envoy build takes place. +export ENVOY_BUILD_DIR=/tmp/envoy-docker-build + +# Do a build matrix with different types of builds docs, coverage, bazel.release, etc. +if [ "$TEST_TYPE" == "docs" ] then echo "docs build..." - make docs + ./docs/build.sh ./docs/publish.sh exit 0 else - docker run -t -i -v /tmp/envoy-docker-build:/build -v $TRAVIS_BUILD_DIR:/source \ + docker run -t -i -v "$ENVOY_BUILD_DIR":/build -v $TRAVIS_BUILD_DIR:/source \ lyft/envoy-build:$ENVOY_BUILD_SHA /bin/bash -c "cd /source && ci/do_ci.sh $TEST_TYPE" fi -# The following scripts are only relevant on a `normal` run. -# This script build a lyft/envoy image an that image is pushed on merge to master. -./ci/docker_push.sh -# This script runs on every PRs normal run to test the docker examples. -./ci/verify_examples.sh +if [ "$TEST_TYPE" == "bazel.release" ] +then + mkdir -p build_release + cp -f "$ENVOY_BUILD_DIR"/envoy/source/exe/envoy ./build_release + # This script builds a lyft/envoy image and pushes that image on merge to master. + ./ci/docker_push.sh + # This script runs on every PRs release run to test the docker examples. + ./ci/verify_examples.sh +fi diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 0eea618e87b00..5f88335a0c85c 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -7,15 +7,44 @@ set -e . "$(dirname "$0")"/build_setup.sh echo "building using ${NUM_CPUS} CPUs" -if [[ "$1" == "bazel.debug" ]]; then - echo "bazel debug build with tests..." +if [[ "$1" == "bazel.release" ]]; then + echo "bazel release build with tests..." cd "${ENVOY_CONSUMER_SRCDIR}" echo "Building..." - bazel build ${BAZEL_BUILD_OPTIONS} @envoy//source/exe:envoy-static.stamped + bazel --batch build ${BAZEL_BUILD_OPTIONS} -c opt @envoy//source/exe:envoy-static.stripped.stamped + # Copy the envoy-static binary somewhere that we can access outside of the + # container. + DELIVER_DIR="${ENVOY_BUILD_DIR}"/source/exe + mkdir -p "${DELIVER_DIR}" + cp -f \ + "${ENVOY_CONSUMER_SRCDIR}"/bazel-genfiles/external/envoy/source/exe/envoy-static.stripped.stamped \ + "${DELIVER_DIR}"/envoy echo "Testing..." - bazel test ${BAZEL_TEST_OPTIONS} --test_output=all \ + bazel --batch test ${BAZEL_TEST_OPTIONS} -c opt --test_output=all \ --cache_test_results=no @envoy//test/... //:echo2_integration_test exit 0 +elif [[ "$1" == "bazel.asan" ]]; then + echo "bazel ASAN debug build with tests..." + cd "${ENVOY_CONSUMER_SRCDIR}" + echo "Building and testing..." + # TODO(htuch): This should switch to using clang when available. + bazel --batch test ${BAZEL_TEST_OPTIONS} -c dbg --config=asan --test_output=all \ + --cache_test_results=no @envoy//test/... //:echo2_integration_test + exit 0 +elif [[ "$1" == "bazel.fastbuild" ]]; then + # This doesn't go into CI but is available for developer convenience. + echo "bazel fastbuild of envoy-static..." + cd "${ENVOY_BUILD_DIR}" + echo "Building..." + bazel build ${BAZEL_BUILD_OPTIONS} -c fastbuild //source/exe:envoy-static + exit 0 +elif [[ "$1" == "bazel.fastbuild.test" ]]; then + # This doesn't go into CI but is available for developer convenience. + echo "bazel test with fastbuild..." + cd "${ENVOY_BUILD_DIR}" + echo "Building and testing..." + bazel test ${BAZEL_BUILD_OPTIONS} -c fastbuild //test/... + exit 0 elif [[ "$1" == "bazel.coverage" ]]; then echo "bazel coverage build with tests..." export GCOVR="/thirdparty/gcovr-3.3/scripts/gcovr" @@ -32,6 +61,7 @@ elif [[ "$1" == "bazel.coverage" ]]; then # some Bazel created symlinks to the source directory in its output # directory. Wow. cd "${ENVOY_BUILD_DIR}" + export BAZEL_TEST_OPTIONS="${BAZEL_TEST_OPTIONS} -c dbg" SRCDIR="${GCOVR_DIR}" "${ENVOY_SRCDIR}"/test/run_envoy_bazel_coverage.sh exit 0 elif [[ "$1" == "fix_format" ]]; then @@ -44,25 +74,7 @@ elif [[ "$1" == "check_format" ]]; then cd "${ENVOY_SRCDIR}" ./tools/check_format.py check exit 0 -elif [[ "$1" == "coverage" ]]; then - echo "coverage build with tests..." - TEST_TARGET="envoy.check-coverage" -elif [[ "$1" == "asan" ]]; then - echo "asan build with tests..." - TEST_TARGET="envoy.check" -elif [[ "$1" == "debug" ]]; then - echo "debug build with tests..." - TEST_TARGET="envoy.check" -elif [[ "$1" == "server_only" ]]; then - echo "normal build server only..." - TEST_TARGET="envoy" else - echo "normal build with tests..." - TEST_TARGET="envoy.check" + echo "Invalid do_ci.sh target, see ci/README.md for valid targets." + exit 1 fi - -shift -export EXTRA_TEST_ARGS="$@" - -[[ "${SKIP_CHECK_FORMAT}" == "1" ]] || make check_format -make -j${NUM_CPUS} ${TEST_TARGET} diff --git a/ci/docker_push.sh b/ci/docker_push.sh index d746b55d7968a..7aad176cc292f 100755 --- a/ci/docker_push.sh +++ b/ci/docker_push.sh @@ -1,35 +1,33 @@ #!/bin/bash + set -e set -x -if [ "$TEST_TYPE" == "normal" ] -then - # this is needed to verify the example images - docker build -f ci/Dockerfile-envoy-image -t lyft/envoy:latest . +# this is needed to verify the example images +docker build -f ci/Dockerfile-envoy-image -t lyft/envoy:latest . - # push the envoy image on merge to master - want_push='false' - for branch in "master" - do - if [ "$TRAVIS_BRANCH" == "$branch" ] - then - want_push='true' - fi - done - if [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$want_push" == "true" ] - then - docker login -e $DOCKER_EMAIL -u $DOCKER_USERNAME -p $DOCKER_PASSWORD - docker push lyft/envoy:latest - docker tag lyft/envoy:latest lyft/envoy:$TRAVIS_COMMIT - docker push lyft/envoy:$TRAVIS_COMMIT - make -C ci/build_alpine_container - docker tag lyft/envoy-alpine:latest lyft/envoy-alpine:$TRAVIS_COMMIT - docker push lyft/envoy-alpine:$TRAVIS_COMMIT - docker push lyft/envoy-alpine:latest - docker tag lyft/envoy-alpine-debug:latest lyft/envoy-alpine-debug:$TRAVIS_COMMIT - docker push lyft/envoy-alpine-debug:$TRAVIS_COMMIT - docker push lyft/envoy-alpine-debug:latest - else - echo 'Ignoring PR branch for docker push.' - fi +# push the envoy image on merge to master +want_push='false' +for branch in "master" +do + if [ "$TRAVIS_BRANCH" == "$branch" ] + then + want_push='true' + fi +done +if [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$want_push" == "true" ] +then + docker login -e $DOCKER_EMAIL -u $DOCKER_USERNAME -p $DOCKER_PASSWORD + docker push lyft/envoy:latest + docker tag lyft/envoy:latest lyft/envoy:$TRAVIS_COMMIT + docker push lyft/envoy:$TRAVIS_COMMIT + make -C ci/build_alpine_container + docker tag lyft/envoy-alpine:latest lyft/envoy-alpine:$TRAVIS_COMMIT + docker push lyft/envoy-alpine:$TRAVIS_COMMIT + docker push lyft/envoy-alpine:latest + docker tag lyft/envoy-alpine-debug:latest lyft/envoy-alpine-debug:$TRAVIS_COMMIT + docker push lyft/envoy-alpine-debug:$TRAVIS_COMMIT + docker push lyft/envoy-alpine-debug:latest +else + echo 'Ignoring PR branch for docker push.' fi diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index ec95a1f3f9b2e..50678a657ff2e 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -3,9 +3,9 @@ set -e [[ -z "${IMAGE_ID}" ]] && IMAGE_ID="latest" +[[ -z "${ENVOY_DOCKER_BUILD_DIR}" ]] && ENVOY_DOCKER_BUILD_DIR=/tmp/envoy-docker-build -BUILD_DIR=/tmp/envoy-docker-build -mkdir -p "${BUILD_DIR}" +mkdir -p "${ENVOY_DOCKER_BUILD_DIR}" docker pull lyft/envoy-build:"${IMAGE_ID}" -docker run -t -i -u $(id -u):$(id -g) -v "${BUILD_DIR}":/build \ +docker run -t -i -u $(id -u):$(id -g) -v "${ENVOY_DOCKER_BUILD_DIR}":/build \ -v "$PWD":/source lyft/envoy-build:"${IMAGE_ID}" /bin/bash -c "cd source && $*" diff --git a/ci/verify_examples.sh b/ci/verify_examples.sh index 259de66c7dcd5..14133b91ec6e1 100755 --- a/ci/verify_examples.sh +++ b/ci/verify_examples.sh @@ -1,5 +1,7 @@ #!/bin/bash + set -e + verify() { echo $1 CONTAINER_ID="$(docker ps -aqf name=$1)" @@ -10,36 +12,31 @@ verify() { fi } -if [ "$TEST_TYPE" == "normal" ] -then - # Test front proxy example - cd examples/front-proxy - docker-compose up --build -d - for CONTAINER_NAME in "frontproxy_front-envoy" "frontproxy_service1" "frontproxy_service2" - do - verify $CONTAINER_NAME - done - cd ../ +# Test front proxy example +cd examples/front-proxy +docker-compose up --build -d +for CONTAINER_NAME in "frontproxy_front-envoy" "frontproxy_service1" "frontproxy_service2" +do + verify $CONTAINER_NAME +done +cd ../ - # Test grpc bridge example - # install go - curl -O https://storage.googleapis.com/golang/go1.7.1.linux-amd64.tar.gz - tar -xf go1.7.1.linux-amd64.tar.gz - sudo mv go /usr/local - export PATH=$PATH:/usr/local/go/bin - export GOPATH=$HOME/go - mkdir -p $GOPATH/src/github.com/lyft/envoy/examples/ - cp -r grpc-bridge $GOPATH/src/github.com/lyft/envoy/examples/ - # build example - cd $GOPATH/src/github.com/lyft/envoy/examples/grpc-bridge - ./script/bootstrap - ./script/build - # verify example works - docker-compose up --build -d - for CONTAINER_NAME in "grpcbridge_python" "grpcbridge_grpc" - do - verify $CONTAINER_NAME - done -else - echo 'Ignoring non-normal build branch' -fi +# Test grpc bridge example +# install go +curl -O https://storage.googleapis.com/golang/go1.7.1.linux-amd64.tar.gz +tar -xf go1.7.1.linux-amd64.tar.gz +sudo mv go /usr/local +export PATH=$PATH:/usr/local/go/bin +export GOPATH=$HOME/go +mkdir -p $GOPATH/src/github.com/lyft/envoy/examples/ +cp -r grpc-bridge $GOPATH/src/github.com/lyft/envoy/examples/ +# build example +cd $GOPATH/src/github.com/lyft/envoy/examples/grpc-bridge +./script/bootstrap +./script/build +# verify example works +docker-compose up --build -d +for CONTAINER_NAME in "grpcbridge_python" "grpcbridge_grpc" +do + verify $CONTAINER_NAME +done diff --git a/docs/README.md b/docs/README.md index 49ac9d7cb3c81..800238a8af831 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,3 +1,13 @@ +# Developer-local docs build + +```bash +./docs/build.sh +``` + +The output can be found in `generated/docs`. + +# How the Envoy website and docs are updated + The Envoy website, and docs are automatically built, and pushed on every commit to master. This process is handled by Travis CI with the [`publish.sh`](https://github.com/lyft/envoy/blob/master/docs/publish.sh) script. diff --git a/docs/build.sh b/docs/build.sh index 9c20244d3d668..83a733e1642d4 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -1,13 +1,17 @@ #!/bin/bash -SCRIPT_DIR=`dirname $0` +SCRIPT_DIR=$(dirname "$0") BUILD_DIR=build/docs +[[ -z "${DOCS_OUTPUT_DIR}" ]] && DOCS_OUTPUT_DIR=generated/docs -if [ ! -d $BUILD_DIR/venv ]; then - virtualenv $BUILD_DIR/venv --no-site-packages - $BUILD_DIR/venv/bin/pip install -r $SCRIPT_DIR/requirements.txt +rm -rf "${DOCS_OUTPUT_DIR}" +mkdir -p "${DOCS_OUTPUT_DIR}" + +if [ ! -d "${BUILD_DIR}"/venv ]; then + virtualenv "${BUILD_DIR}"/venv --no-site-packages + "${BUILD_DIR}"/venv/bin/pip install -r "${SCRIPT_DIR}"/requirements.txt fi -source $BUILD_DIR/venv/bin/activate -cp -r $SCRIPT_DIR/landing_generated/* $1 -sphinx-build -W -b html $SCRIPT_DIR $1/docs +source "${BUILD_DIR}"/venv/bin/activate +cp -r "${SCRIPT_DIR}"/landing_generated/* "${DOCS_OUTPUT_DIR}" +sphinx-build -W -b html "${SCRIPT_DIR}" "${DOCS_OUTPUT_DIR}"/docs diff --git a/test/run_envoy_bazel_coverage.sh b/test/run_envoy_bazel_coverage.sh index fd59d0f989dcc..0726ddbca5535 100755 --- a/test/run_envoy_bazel_coverage.sh +++ b/test/run_envoy_bazel_coverage.sh @@ -29,7 +29,7 @@ echo "Cleanup completed." # https://github.com/bazelbuild/bazel/issues/1118). This works today as we have # a single coverage test binary and do not require the "bazel coverage" support # for collecting multiple traces and glueing them together. -"${BAZEL_COVERAGE}" test "${COVERAGE_TARGET}" ${BAZEL_TEST_OPTIONS} \ +"${BAZEL_COVERAGE}" --batch test "${COVERAGE_TARGET}" ${BAZEL_TEST_OPTIONS} \ --cache_test_results=no --cxxopt="--coverage" --linkopt="--coverage" \ --test_output=all --strategy=Genrule=standalone --spawn_strategy=standalone