ci: refactor external dep build/install scripts.

ci/build_container/Dockerfile

@@ -1,3 +1,4 @@
 FROM ubuntu:xenial
-COPY ./build_container.sh /
+COPY ./build_container.sh ./build_and_install_deps.sh ./Makefile /
+COPY ./build_recipes/*.sh /build_recipes/
 RUN ./build_container.sh

ci/build_container/Makefile

@@ -0,0 +1,44 @@
+# The individual build recipe scripts must contain sufficient information (e.g.  SHA, URL, repo,
+# version number, etc.) to uniquely identify the revision of the upstream dependency.  This allows
+# make to pick up changes with a simple direct dependency on the build recipe.
+# TODO(htuch): These dependencies should be installed into distinct --prefix directories, so we can
+# easily blow away the lib/ and include/ install directories when we upgrade version.
+
+all: $(THIRDPARTY_DEPS)/libevent.dep \
+	$(THIRDPARTY_DEPS)/boringssl.dep \
+	$(THIRDPARTY_DEPS)/gperftools.dep \
+	$(THIRDPARTY_DEPS)/nghttp2.dep \
+	$(THIRDPARTY_DEPS)/cares.dep \
+	$(THIRDPARTY_DEPS)/protobuf.dep \
+	$(THIRDPARTY_DEPS)/cotire.dep \
+	$(THIRDPARTY_DEPS)/spdlog.dep \
+	$(THIRDPARTY_DEPS)/http-parser.dep \
+	$(THIRDPARTY_DEPS)/tclap.dep \
+	$(THIRDPARTY_DEPS)/lightstep.dep \
+	$(THIRDPARTY_DEPS)/rapidjson.dep \
+	$(THIRDPARTY_DEPS)/googletest.dep \
+	$(THIRDPARTY_DEPS)/gcovr.dep
+
+RECIPES := build_recipes
+
+build-setup = rm -rf $@.build && \
+	mkdir -p $@.build && \
+	mkdir -p $(THIRDPARTY_BUILD)/lib && \
+	mkdir -p $(THIRDPARTY_BUILD)/include && \
+	cd $@.build && \
+	echo "Building in $@.build, logs at $@.log"
+
+build-complete = rm -rf $@.build && \
+	echo "Successful build of $@" && \
+	touch $@
+
+build-recipe = cd $(THIRDPARTY_SRC) && \
+	$(build-setup) && \
+	(((bash $(realpath $<) 2>&1) > $@.log) || (cat $@.log; exit 1)) && \
+	$(build-complete)
+
+$(THIRDPARTY_DEPS)/%.dep: $(RECIPES)/%.sh
+	@+$(build-recipe)
+
+$(THIRDPARTY_DEPS)/lightstep.dep: $(RECIPES)/lightstep.sh $(THIRDPARTY_DEPS)/protobuf.dep
+	@+$(build-recipe)

ci/build_container/build_and_install_deps.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e
+
+mkdir -p "${THIRDPARTY_DEPS}"
+mkdir -p "${THIRDPARTY_BUILD}"
+mkdir -p "${THIRDPARTY_SRC}"
+
+NUM_CPUS=`grep -c ^processor /proc/cpuinfo`
+make -C "$(dirname "$0")" -j "${NUM_CPUS}"

ci/build_container/build_container.sh

@@ -26,127 +26,11 @@ pip install virtualenv
 export GOPATH=/usr/lib/go
 go get github.com/bazelbuild/buildifier/buildifier
 
-# Build artifacts
-THIRDPARTY_BUILD=/thirdparty_build
-mkdir $THIRDPARTY_BUILD
-
-# Source artifacts
-mkdir thirdparty
-cd thirdparty
-
 # GCC 4.9 for everything.
 export CC=gcc-4.9
 export CXX=g++-4.9
 
-# libevent
-wget https://github.com/libevent/libevent/releases/download/release-2.1.8-stable/libevent-2.1.8-stable.tar.gz
-tar xf libevent-2.1.8-stable.tar.gz
-cd libevent-2.1.8-stable
-./configure --prefix=$THIRDPARTY_BUILD --enable-shared=no --disable-libevent-regress --disable-openssl
-make install
-cd ..
-rm -fr libevent*
-
-# BoringSSL
-git clone https://boringssl.googlesource.com/boringssl
-cd boringssl
-git reset --hard b87c80300647c2c0311c1489a104470e099f1531
-cmake .
-make
-cp -r include/* $THIRDPARTY_BUILD/include
-cp ssl/libssl.a $THIRDPARTY_BUILD/lib
-cp crypto/libcrypto.a $THIRDPARTY_BUILD/lib
-cd ..
-rm -rf boringssl
-
-# gperftools
-wget https://github.com/gperftools/gperftools/releases/download/gperftools-2.5/gperftools-2.5.tar.gz
-tar xf gperftools-2.5.tar.gz
-cd gperftools-2.5
-./configure --prefix=$THIRDPARTY_BUILD --enable-shared=no --enable-frame-pointers
-make install
-cd ..
-rm -fr gperftools*
-
-# nghttp2
-wget https://github.com/nghttp2/nghttp2/releases/download/v1.20.0/nghttp2-1.20.0.tar.gz
-tar xf nghttp2-1.20.0.tar.gz
-cd nghttp2-1.20.0
-./configure --prefix=$THIRDPARTY_BUILD --enable-shared=no --enable-lib-only
-make install
-cd ..
-rm -fr nghttp2*
-
-# c-ares
-wget https://github.com/c-ares/c-ares/archive/cares-1_12_0.tar.gz
-tar xf cares-1_12_0.tar.gz
-cd c-ares-cares-1_12_0
-./buildconf
-./configure --prefix=$THIRDPARTY_BUILD --enable-shared=no --enable-lib-only
-make install
-cd ..
-rm -fr cares* c-ares*
-
-# protobuf
-wget https://github.com/google/protobuf/releases/download/v3.2.0/protobuf-cpp-3.2.0.tar.gz
-tar xf protobuf-cpp-3.2.0.tar.gz
-cd protobuf-3.2.0
-./configure --prefix=$THIRDPARTY_BUILD --enable-shared=no
-make install
-make distclean
-cd ..
-
-# cotire
-wget https://github.com/sakra/cotire/archive/cotire-1.7.8.tar.gz
-tar xf cotire-1.7.8.tar.gz
-rm cotire-1.7.8.tar.gz
-
-# spdlog
-wget https://github.com/gabime/spdlog/archive/v0.11.0.tar.gz
-tar xf v0.11.0.tar.gz
-rm v0.11.0.tar.gz
-
-# http-parser
-wget -O http-parser-v2.7.0.tar.gz https://github.com/nodejs/http-parser/archive/v2.7.0.tar.gz
-tar xf http-parser-v2.7.0.tar.gz
-cd http-parser-2.7.0
-$CC -O2 -c http_parser.c -o http_parser.o
-ar rcs libhttp_parser.a http_parser.o
-cp libhttp_parser.a $THIRDPARTY_BUILD/lib
-cp http_parser.h $THIRDPARTY_BUILD/include
-cd ..
-rm -fr http-parser*
-
-# tclap
-wget -O tclap-1.2.1.tar.gz https://sourceforge.net/projects/tclap/files/tclap-1.2.1.tar.gz/download
-tar xf tclap-1.2.1.tar.gz
-rm tclap-1.2.1.tar.gz
-
-# lightstep
-wget https://github.com/lightstep/lightstep-tracer-cpp/releases/download/v0_36/lightstep-tracer-cpp-0.36.tar.gz
-tar xf lightstep-tracer-cpp-0.36.tar.gz
-cd lightstep-tracer-cpp-0.36
-./configure --disable-grpc --prefix=$THIRDPARTY_BUILD --enable-shared=no \
-	    protobuf_CFLAGS="-I$THIRDPARTY_BUILD/include" protobuf_LIBS="-L$THIRDPARTY_BUILD/lib -lprotobuf" PROTOC=$THIRDPARTY_BUILD/bin/protoc
-make install
-cd ..
-rm -rf lightstep-tracer*
-
-# rapidjson
-wget -O rapidjson-1.1.0.tar.gz https://github.com/miloyip/rapidjson/archive/v1.1.0.tar.gz
-tar xf rapidjson-1.1.0.tar.gz
-rm rapidjson-1.1.0.tar.gz
-
-# googletest
-wget -O googletest-1.8.0.tar.gz https://github.com/google/googletest/archive/release-1.8.0.tar.gz
-tar xf googletest-1.8.0.tar.gz
-cd googletest-release-1.8.0
-cmake -DCMAKE_INSTALL_PREFIX:PATH=$THIRDPARTY_BUILD .
-make install
-cd ..
-rm -fr googletest*
-
-# gcovr
-wget -O gcovr-3.3.tar.gz https://github.com/gcovr/gcovr/archive/3.3.tar.gz
-tar xf gcovr-3.3.tar.gz
-rm gcovr-3.3.tar.gz
+export THIRDPARTY_DEPS=/tmp
+export THIRDPARTY_SRC=/thirdparty
+export THIRDPARTY_BUILD=/thirdparty_build
+"$(dirname "$0")"/build_and_install_deps.sh

ci/build_container/build_recipes/boringssl.sh

@@ -0,0 +1,8 @@
+git clone https://boringssl.googlesource.com/boringssl
+cd boringssl
+git reset --hard b87c80300647c2c0311c1489a104470e099f1531
+cmake .
+make
+cp -r include/* $THIRDPARTY_BUILD/include
+cp ssl/libssl.a $THIRDPARTY_BUILD/lib
+cp crypto/libcrypto.a $THIRDPARTY_BUILD/lib

ci/build_container/build_recipes/cares.sh

@@ -0,0 +1,6 @@
+wget https://github.com/c-ares/c-ares/archive/cares-1_12_0.tar.gz
+tar xf cares-1_12_0.tar.gz
+cd c-ares-cares-1_12_0
+./buildconf
+./configure --prefix=$THIRDPARTY_BUILD --enable-shared=no --enable-lib-only
+make install

ci/build_container/build_recipes/cotire.sh

@@ -0,0 +1,3 @@
+wget https://github.com/sakra/cotire/archive/cotire-1.7.8.tar.gz
+tar xf cotire-1.7.8.tar.gz
+rsync -av cotire-cotire-1.7.8 $THIRDPARTY_SRC

ci/build_container/build_recipes/gcovr.sh

@@ -0,0 +1,4 @@
+wget -O gcovr-3.3.tar.gz https://github.com/gcovr/gcovr/archive/3.3.tar.gz
+tar xf gcovr-3.3.tar.gz
+rsync -av gcovr-3.3 $THIRDPARTY_SRC
+rm gcovr-3.3.tar.gz

ci/build_container/build_recipes/googletest.sh

@@ -0,0 +1,5 @@
+wget -O googletest-1.8.0.tar.gz https://github.com/google/googletest/archive/release-1.8.0.tar.gz
+tar xf googletest-1.8.0.tar.gz
+cd googletest-release-1.8.0
+cmake -DCMAKE_INSTALL_PREFIX:PATH=$THIRDPARTY_BUILD .
+make install

ci/build_container/build_recipes/gperftools.sh

@@ -0,0 +1,5 @@
+wget https://github.com/gperftools/gperftools/releases/download/gperftools-2.5/gperftools-2.5.tar.gz
+tar xf gperftools-2.5.tar.gz
+cd gperftools-2.5
+./configure --prefix=$THIRDPARTY_BUILD --enable-shared=no --enable-frame-pointers
+make install

ci/build_container/build_recipes/http-parser.sh

@@ -0,0 +1,7 @@
+wget -O http-parser-v2.7.0.tar.gz https://github.com/nodejs/http-parser/archive/v2.7.0.tar.gz
+tar xf http-parser-v2.7.0.tar.gz
+cd http-parser-2.7.0
+$CC -O2 -c http_parser.c -o http_parser.o
+ar rcs libhttp_parser.a http_parser.o
+cp libhttp_parser.a $THIRDPARTY_BUILD/lib
+cp http_parser.h $THIRDPARTY_BUILD/include

ci/build_container/build_recipes/libevent.sh

@@ -0,0 +1,5 @@
+wget -o - https://github.com/libevent/libevent/releases/download/release-2.1.8-stable/libevent-2.1.8-stable.tar.gz
+tar xf libevent-2.1.8-stable.tar.gz
+cd libevent-2.1.8-stable
+./configure --prefix=$THIRDPARTY_BUILD --enable-shared=no --disable-libevent-regress --disable-openssl
+make install

ci/build_container/build_recipes/lightstep.sh

@@ -0,0 +1,6 @@
+wget https://github.com/lightstep/lightstep-tracer-cpp/releases/download/v0_36/lightstep-tracer-cpp-0.36.tar.gz
+tar xf lightstep-tracer-cpp-0.36.tar.gz
+cd lightstep-tracer-cpp-0.36
+./configure --disable-grpc --prefix=$THIRDPARTY_BUILD --enable-shared=no \
+	    protobuf_CFLAGS="-I$THIRDPARTY_BUILD/include" protobuf_LIBS="-L$THIRDPARTY_BUILD/lib -lprotobuf" PROTOC=$THIRDPARTY_BUILD/bin/protoc
+make install

ci/build_container/build_recipes/nghttp2.sh

@@ -0,0 +1,5 @@
+wget https://github.com/nghttp2/nghttp2/releases/download/v1.20.0/nghttp2-1.20.0.tar.gz
+tar xf nghttp2-1.20.0.tar.gz
+cd nghttp2-1.20.0
+./configure --prefix=$THIRDPARTY_BUILD --enable-shared=no --enable-lib-only
+make install

ci/build_container/build_recipes/protobuf.sh

@@ -0,0 +1,6 @@
+wget https://github.com/google/protobuf/releases/download/v3.2.0/protobuf-cpp-3.2.0.tar.gz
+tar xf protobuf-cpp-3.2.0.tar.gz
+rsync -av protobuf-3.2.0 $THIRDPARTY_SRC
+cd protobuf-3.2.0
+./configure --prefix=$THIRDPARTY_BUILD --enable-shared=no
+make install

ci/build_container/build_recipes/rapidjson.sh

@@ -0,0 +1,3 @@
+wget -O rapidjson-1.1.0.tar.gz https://github.com/miloyip/rapidjson/archive/v1.1.0.tar.gz
+tar xf rapidjson-1.1.0.tar.gz
+rsync -av rapidjson-1.1.0 $THIRDPARTY_SRC

ci/build_container/build_recipes/spdlog.sh

@@ -0,0 +1,3 @@
+wget https://github.com/gabime/spdlog/archive/v0.11.0.tar.gz
+tar xf v0.11.0.tar.gz
+rsync -av spdlog-0.11.0 $THIRDPARTY_SRC

ci/build_container/build_recipes/tclap.sh

@@ -0,0 +1,3 @@
+wget -O tclap-1.2.1.tar.gz https://sourceforge.net/projects/tclap/files/tclap-1.2.1.tar.gz/download
+tar xf tclap-1.2.1.tar.gz
+rsync -av tclap-1.2.1 $THIRDPARTY_SRC