diff --git a/ci/build_container/Dockerfile b/ci/build_container/Dockerfile index 29c3e901944a6..aaafd91ff0673 100644 --- a/ci/build_container/Dockerfile +++ b/ci/build_container/Dockerfile @@ -1,3 +1,4 @@ FROM ubuntu:xenial -COPY ./build_container.sh / +COPY ./build_container.sh ./build_and_install_deps.sh ./Makefile / +COPY ./build_recipes/*.sh /build_recipes/ RUN ./build_container.sh diff --git a/ci/build_container/Makefile b/ci/build_container/Makefile new file mode 100644 index 0000000000000..a0518a312c22d --- /dev/null +++ b/ci/build_container/Makefile @@ -0,0 +1,44 @@ +# The individual build recipe scripts must contain sufficient information (e.g. SHA, URL, repo, +# version number, etc.) to uniquely identify the revision of the upstream dependency. This allows +# make to pick up changes with a simple direct dependency on the build recipe. +# TODO(htuch): These dependencies should be installed into distinct --prefix directories, so we can +# easily blow away the lib/ and include/ install directories when we upgrade version. + +all: $(THIRDPARTY_DEPS)/libevent.dep \ + $(THIRDPARTY_DEPS)/boringssl.dep \ + $(THIRDPARTY_DEPS)/gperftools.dep \ + $(THIRDPARTY_DEPS)/nghttp2.dep \ + $(THIRDPARTY_DEPS)/cares.dep \ + $(THIRDPARTY_DEPS)/protobuf.dep \ + $(THIRDPARTY_DEPS)/cotire.dep \ + $(THIRDPARTY_DEPS)/spdlog.dep \ + $(THIRDPARTY_DEPS)/http-parser.dep \ + $(THIRDPARTY_DEPS)/tclap.dep \ + $(THIRDPARTY_DEPS)/lightstep.dep \ + $(THIRDPARTY_DEPS)/rapidjson.dep \ + $(THIRDPARTY_DEPS)/googletest.dep \ + $(THIRDPARTY_DEPS)/gcovr.dep + +RECIPES := build_recipes + +build-setup = rm -rf $@.build && \ + mkdir -p $@.build && \ + mkdir -p $(THIRDPARTY_BUILD)/lib && \ + mkdir -p $(THIRDPARTY_BUILD)/include && \ + cd $@.build && \ + echo "Building in $@.build, logs at $@.log" + +build-complete = rm -rf $@.build && \ + echo "Successful build of $@" && \ + touch $@ + +build-recipe = cd $(THIRDPARTY_SRC) && \ + $(build-setup) && \ + (((bash $(realpath $<) 2>&1) > $@.log) || (cat $@.log; exit 1)) && \ + $(build-complete) + +$(THIRDPARTY_DEPS)/%.dep: $(RECIPES)/%.sh + @+$(build-recipe) + +$(THIRDPARTY_DEPS)/lightstep.dep: $(RECIPES)/lightstep.sh $(THIRDPARTY_DEPS)/protobuf.dep + @+$(build-recipe) diff --git a/ci/build_container/build_and_install_deps.sh b/ci/build_container/build_and_install_deps.sh new file mode 100755 index 0000000000000..ad026f19ed0fd --- /dev/null +++ b/ci/build_container/build_and_install_deps.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +mkdir -p "${THIRDPARTY_DEPS}" +mkdir -p "${THIRDPARTY_BUILD}" +mkdir -p "${THIRDPARTY_SRC}" + +NUM_CPUS=`grep -c ^processor /proc/cpuinfo` +make -C "$(dirname "$0")" -j "${NUM_CPUS}" diff --git a/ci/build_container/build_container.sh b/ci/build_container/build_container.sh index 35a80fa93add3..003fe7957bc40 100755 --- a/ci/build_container/build_container.sh +++ b/ci/build_container/build_container.sh @@ -26,127 +26,11 @@ pip install virtualenv export GOPATH=/usr/lib/go go get github.com/bazelbuild/buildifier/buildifier -# Build artifacts -THIRDPARTY_BUILD=/thirdparty_build -mkdir $THIRDPARTY_BUILD - -# Source artifacts -mkdir thirdparty -cd thirdparty - # GCC 4.9 for everything. export CC=gcc-4.9 export CXX=g++-4.9 -# libevent -wget https://github.com/libevent/libevent/releases/download/release-2.1.8-stable/libevent-2.1.8-stable.tar.gz -tar xf libevent-2.1.8-stable.tar.gz -cd libevent-2.1.8-stable -./configure --prefix=$THIRDPARTY_BUILD --enable-shared=no --disable-libevent-regress --disable-openssl -make install -cd .. -rm -fr libevent* - -# BoringSSL -git clone https://boringssl.googlesource.com/boringssl -cd boringssl -git reset --hard b87c80300647c2c0311c1489a104470e099f1531 -cmake . -make -cp -r include/* $THIRDPARTY_BUILD/include -cp ssl/libssl.a $THIRDPARTY_BUILD/lib -cp crypto/libcrypto.a $THIRDPARTY_BUILD/lib -cd .. -rm -rf boringssl - -# gperftools -wget https://github.com/gperftools/gperftools/releases/download/gperftools-2.5/gperftools-2.5.tar.gz -tar xf gperftools-2.5.tar.gz -cd gperftools-2.5 -./configure --prefix=$THIRDPARTY_BUILD --enable-shared=no --enable-frame-pointers -make install -cd .. -rm -fr gperftools* - -# nghttp2 -wget https://github.com/nghttp2/nghttp2/releases/download/v1.20.0/nghttp2-1.20.0.tar.gz -tar xf nghttp2-1.20.0.tar.gz -cd nghttp2-1.20.0 -./configure --prefix=$THIRDPARTY_BUILD --enable-shared=no --enable-lib-only -make install -cd .. -rm -fr nghttp2* - -# c-ares -wget https://github.com/c-ares/c-ares/archive/cares-1_12_0.tar.gz -tar xf cares-1_12_0.tar.gz -cd c-ares-cares-1_12_0 -./buildconf -./configure --prefix=$THIRDPARTY_BUILD --enable-shared=no --enable-lib-only -make install -cd .. -rm -fr cares* c-ares* - -# protobuf -wget https://github.com/google/protobuf/releases/download/v3.2.0/protobuf-cpp-3.2.0.tar.gz -tar xf protobuf-cpp-3.2.0.tar.gz -cd protobuf-3.2.0 -./configure --prefix=$THIRDPARTY_BUILD --enable-shared=no -make install -make distclean -cd .. - -# cotire -wget https://github.com/sakra/cotire/archive/cotire-1.7.8.tar.gz -tar xf cotire-1.7.8.tar.gz -rm cotire-1.7.8.tar.gz - -# spdlog -wget https://github.com/gabime/spdlog/archive/v0.11.0.tar.gz -tar xf v0.11.0.tar.gz -rm v0.11.0.tar.gz - -# http-parser -wget -O http-parser-v2.7.0.tar.gz https://github.com/nodejs/http-parser/archive/v2.7.0.tar.gz -tar xf http-parser-v2.7.0.tar.gz -cd http-parser-2.7.0 -$CC -O2 -c http_parser.c -o http_parser.o -ar rcs libhttp_parser.a http_parser.o -cp libhttp_parser.a $THIRDPARTY_BUILD/lib -cp http_parser.h $THIRDPARTY_BUILD/include -cd .. -rm -fr http-parser* - -# tclap -wget -O tclap-1.2.1.tar.gz https://sourceforge.net/projects/tclap/files/tclap-1.2.1.tar.gz/download -tar xf tclap-1.2.1.tar.gz -rm tclap-1.2.1.tar.gz - -# lightstep -wget https://github.com/lightstep/lightstep-tracer-cpp/releases/download/v0_36/lightstep-tracer-cpp-0.36.tar.gz -tar xf lightstep-tracer-cpp-0.36.tar.gz -cd lightstep-tracer-cpp-0.36 -./configure --disable-grpc --prefix=$THIRDPARTY_BUILD --enable-shared=no \ - protobuf_CFLAGS="-I$THIRDPARTY_BUILD/include" protobuf_LIBS="-L$THIRDPARTY_BUILD/lib -lprotobuf" PROTOC=$THIRDPARTY_BUILD/bin/protoc -make install -cd .. -rm -rf lightstep-tracer* - -# rapidjson -wget -O rapidjson-1.1.0.tar.gz https://github.com/miloyip/rapidjson/archive/v1.1.0.tar.gz -tar xf rapidjson-1.1.0.tar.gz -rm rapidjson-1.1.0.tar.gz - -# googletest -wget -O googletest-1.8.0.tar.gz https://github.com/google/googletest/archive/release-1.8.0.tar.gz -tar xf googletest-1.8.0.tar.gz -cd googletest-release-1.8.0 -cmake -DCMAKE_INSTALL_PREFIX:PATH=$THIRDPARTY_BUILD . -make install -cd .. -rm -fr googletest* - -# gcovr -wget -O gcovr-3.3.tar.gz https://github.com/gcovr/gcovr/archive/3.3.tar.gz -tar xf gcovr-3.3.tar.gz -rm gcovr-3.3.tar.gz +export THIRDPARTY_DEPS=/tmp +export THIRDPARTY_SRC=/thirdparty +export THIRDPARTY_BUILD=/thirdparty_build +"$(dirname "$0")"/build_and_install_deps.sh diff --git a/ci/build_container/build_recipes/boringssl.sh b/ci/build_container/build_recipes/boringssl.sh new file mode 100644 index 0000000000000..ab2ea2b0549ef --- /dev/null +++ b/ci/build_container/build_recipes/boringssl.sh @@ -0,0 +1,8 @@ +git clone https://boringssl.googlesource.com/boringssl +cd boringssl +git reset --hard b87c80300647c2c0311c1489a104470e099f1531 +cmake . +make +cp -r include/* $THIRDPARTY_BUILD/include +cp ssl/libssl.a $THIRDPARTY_BUILD/lib +cp crypto/libcrypto.a $THIRDPARTY_BUILD/lib diff --git a/ci/build_container/build_recipes/cares.sh b/ci/build_container/build_recipes/cares.sh new file mode 100644 index 0000000000000..c3bf8ca0d4258 --- /dev/null +++ b/ci/build_container/build_recipes/cares.sh @@ -0,0 +1,6 @@ +wget https://github.com/c-ares/c-ares/archive/cares-1_12_0.tar.gz +tar xf cares-1_12_0.tar.gz +cd c-ares-cares-1_12_0 +./buildconf +./configure --prefix=$THIRDPARTY_BUILD --enable-shared=no --enable-lib-only +make install diff --git a/ci/build_container/build_recipes/cotire.sh b/ci/build_container/build_recipes/cotire.sh new file mode 100644 index 0000000000000..b0e709ebf13c5 --- /dev/null +++ b/ci/build_container/build_recipes/cotire.sh @@ -0,0 +1,3 @@ +wget https://github.com/sakra/cotire/archive/cotire-1.7.8.tar.gz +tar xf cotire-1.7.8.tar.gz +rsync -av cotire-cotire-1.7.8 $THIRDPARTY_SRC diff --git a/ci/build_container/build_recipes/gcovr.sh b/ci/build_container/build_recipes/gcovr.sh new file mode 100644 index 0000000000000..2b8ec72dfea94 --- /dev/null +++ b/ci/build_container/build_recipes/gcovr.sh @@ -0,0 +1,4 @@ +wget -O gcovr-3.3.tar.gz https://github.com/gcovr/gcovr/archive/3.3.tar.gz +tar xf gcovr-3.3.tar.gz +rsync -av gcovr-3.3 $THIRDPARTY_SRC +rm gcovr-3.3.tar.gz diff --git a/ci/build_container/build_recipes/googletest.sh b/ci/build_container/build_recipes/googletest.sh new file mode 100644 index 0000000000000..a8f7cc1ae719d --- /dev/null +++ b/ci/build_container/build_recipes/googletest.sh @@ -0,0 +1,5 @@ +wget -O googletest-1.8.0.tar.gz https://github.com/google/googletest/archive/release-1.8.0.tar.gz +tar xf googletest-1.8.0.tar.gz +cd googletest-release-1.8.0 +cmake -DCMAKE_INSTALL_PREFIX:PATH=$THIRDPARTY_BUILD . +make install diff --git a/ci/build_container/build_recipes/gperftools.sh b/ci/build_container/build_recipes/gperftools.sh new file mode 100644 index 0000000000000..639de3e21e641 --- /dev/null +++ b/ci/build_container/build_recipes/gperftools.sh @@ -0,0 +1,5 @@ +wget https://github.com/gperftools/gperftools/releases/download/gperftools-2.5/gperftools-2.5.tar.gz +tar xf gperftools-2.5.tar.gz +cd gperftools-2.5 +./configure --prefix=$THIRDPARTY_BUILD --enable-shared=no --enable-frame-pointers +make install diff --git a/ci/build_container/build_recipes/http-parser.sh b/ci/build_container/build_recipes/http-parser.sh new file mode 100644 index 0000000000000..b8d75347a3e0c --- /dev/null +++ b/ci/build_container/build_recipes/http-parser.sh @@ -0,0 +1,7 @@ +wget -O http-parser-v2.7.0.tar.gz https://github.com/nodejs/http-parser/archive/v2.7.0.tar.gz +tar xf http-parser-v2.7.0.tar.gz +cd http-parser-2.7.0 +$CC -O2 -c http_parser.c -o http_parser.o +ar rcs libhttp_parser.a http_parser.o +cp libhttp_parser.a $THIRDPARTY_BUILD/lib +cp http_parser.h $THIRDPARTY_BUILD/include diff --git a/ci/build_container/build_recipes/libevent.sh b/ci/build_container/build_recipes/libevent.sh new file mode 100644 index 0000000000000..48316dc78210d --- /dev/null +++ b/ci/build_container/build_recipes/libevent.sh @@ -0,0 +1,5 @@ +wget -o - https://github.com/libevent/libevent/releases/download/release-2.1.8-stable/libevent-2.1.8-stable.tar.gz +tar xf libevent-2.1.8-stable.tar.gz +cd libevent-2.1.8-stable +./configure --prefix=$THIRDPARTY_BUILD --enable-shared=no --disable-libevent-regress --disable-openssl +make install diff --git a/ci/build_container/build_recipes/lightstep.sh b/ci/build_container/build_recipes/lightstep.sh new file mode 100644 index 0000000000000..5d5efcc1b7128 --- /dev/null +++ b/ci/build_container/build_recipes/lightstep.sh @@ -0,0 +1,6 @@ +wget https://github.com/lightstep/lightstep-tracer-cpp/releases/download/v0_36/lightstep-tracer-cpp-0.36.tar.gz +tar xf lightstep-tracer-cpp-0.36.tar.gz +cd lightstep-tracer-cpp-0.36 +./configure --disable-grpc --prefix=$THIRDPARTY_BUILD --enable-shared=no \ + protobuf_CFLAGS="-I$THIRDPARTY_BUILD/include" protobuf_LIBS="-L$THIRDPARTY_BUILD/lib -lprotobuf" PROTOC=$THIRDPARTY_BUILD/bin/protoc +make install diff --git a/ci/build_container/build_recipes/nghttp2.sh b/ci/build_container/build_recipes/nghttp2.sh new file mode 100644 index 0000000000000..c9b3cb9b53f55 --- /dev/null +++ b/ci/build_container/build_recipes/nghttp2.sh @@ -0,0 +1,5 @@ +wget https://github.com/nghttp2/nghttp2/releases/download/v1.20.0/nghttp2-1.20.0.tar.gz +tar xf nghttp2-1.20.0.tar.gz +cd nghttp2-1.20.0 +./configure --prefix=$THIRDPARTY_BUILD --enable-shared=no --enable-lib-only +make install diff --git a/ci/build_container/build_recipes/protobuf.sh b/ci/build_container/build_recipes/protobuf.sh new file mode 100644 index 0000000000000..d9e7bea21451a --- /dev/null +++ b/ci/build_container/build_recipes/protobuf.sh @@ -0,0 +1,6 @@ +wget https://github.com/google/protobuf/releases/download/v3.2.0/protobuf-cpp-3.2.0.tar.gz +tar xf protobuf-cpp-3.2.0.tar.gz +rsync -av protobuf-3.2.0 $THIRDPARTY_SRC +cd protobuf-3.2.0 +./configure --prefix=$THIRDPARTY_BUILD --enable-shared=no +make install diff --git a/ci/build_container/build_recipes/rapidjson.sh b/ci/build_container/build_recipes/rapidjson.sh new file mode 100644 index 0000000000000..c4bac4f7da8d2 --- /dev/null +++ b/ci/build_container/build_recipes/rapidjson.sh @@ -0,0 +1,3 @@ +wget -O rapidjson-1.1.0.tar.gz https://github.com/miloyip/rapidjson/archive/v1.1.0.tar.gz +tar xf rapidjson-1.1.0.tar.gz +rsync -av rapidjson-1.1.0 $THIRDPARTY_SRC diff --git a/ci/build_container/build_recipes/spdlog.sh b/ci/build_container/build_recipes/spdlog.sh new file mode 100644 index 0000000000000..2677d1bb9ff10 --- /dev/null +++ b/ci/build_container/build_recipes/spdlog.sh @@ -0,0 +1,3 @@ +wget https://github.com/gabime/spdlog/archive/v0.11.0.tar.gz +tar xf v0.11.0.tar.gz +rsync -av spdlog-0.11.0 $THIRDPARTY_SRC diff --git a/ci/build_container/build_recipes/tclap.sh b/ci/build_container/build_recipes/tclap.sh new file mode 100644 index 0000000000000..bde821ed8e3fb --- /dev/null +++ b/ci/build_container/build_recipes/tclap.sh @@ -0,0 +1,3 @@ +wget -O tclap-1.2.1.tar.gz https://sourceforge.net/projects/tclap/files/tclap-1.2.1.tar.gz/download +tar xf tclap-1.2.1.tar.gz +rsync -av tclap-1.2.1 $THIRDPARTY_SRC