Build tsh with static libfido2 in buildbox and Centos7 (#11849)

Build `tsh` with static `libfido2`, `libcbor`,`libcrypto` and `libudev-zero`.

Dockerfiles for buildbox and Centos7 changed. FIPS and macOS to be addressed at
a later date.

Add the `tsh fido2 diag` hidden command for ease of testing.

#9160

* Update go-libfido2 and tidy modules
* Add a fido2 diagnostic command to tsh
* Add a few build artifacts to .gitignore
* Build tsh with static libfido2 in buildbox
* Build tsh with static libfido2 in centos7
* Add a few relevant cmake flags
* Use illiliti/libudev-zero
* Do multi-stage build on centos7, image tweaks
* Add `make enter/centos7`
* s/OFf/OFF/g

.gitignore

@@ -67,5 +67,11 @@ ssh.config
 # Code signing certificate for Windows binaries
 /windows-signing-cert.pfx
 
-# build tooling
-build.assets/tooling/bin/** 
+# Build cache and tooling
+/build.assets/.cache
+/build.assets/tooling/bin/**
+
+# Teleport binaries
+/tctl
+/teleport
+/tsh

Makefile

@@ -156,10 +156,22 @@ endif
 endif
 
 # Enable libfido2 for buildbox and local environments that have it.
-# TODO(codingllama): Build static binaries with libfido2.
 ifneq ("$(shell ls {/opt/homebrew,/usr/lib/x86_64-linux-gnu,/usr/local/lib}/libfido2.* 2>/dev/null)","")
 LIBFIDO2_TAG := libfido2
+else
 endif
+# Build tsh with static libfido2 support if the requirements are present in the
+# system. The required libs are uncommon enough that their presence makes for a
+# strong signal.
+LIBFIDO2_MESSAGE := without libfido2
+ifneq ("$(wildcard /usr/local/lib/libfido2.a)", "")
+ifneq ("$(wildcard /usr/local/lib/libcbor.a)", "")
+ifneq ("$(wildcard /usr/local/lib/libudev.a)", "")
+LIBFIDO2_STATIC_TAG := libfido2 libfido2static
+LIBFIDO2_MESSAGE := with libfido2
+endif # libudev
+endif # libcbor
+endif # libfido2
 
 # Reproducible builds are only available on select targets, and only when OS=linux.
 REPRODUCIBLE ?=
@@ -170,7 +182,7 @@ endif
 # On Windows only build tsh. On all other platforms build teleport, tctl,
 # and tsh.
 BINARIES=$(BUILDDIR)/teleport $(BUILDDIR)/tctl $(BUILDDIR)/tsh $(BUILDDIR)/tbot
-RELEASE_MESSAGE := "Building with GOOS=$(OS) GOARCH=$(ARCH) REPRODUCIBLE=$(REPRODUCIBLE) and $(PAM_MESSAGE) and $(FIPS_MESSAGE) and $(BPF_MESSAGE) and $(ROLETESTER_MESSAGE) and $(RDPCLIENT_MESSAGE)."
+RELEASE_MESSAGE := "Building with GOOS=$(OS) GOARCH=$(ARCH) REPRODUCIBLE=$(REPRODUCIBLE) and $(PAM_MESSAGE) and $(FIPS_MESSAGE) and $(BPF_MESSAGE) and $(ROLETESTER_MESSAGE) and $(RDPCLIENT_MESSAGE) and $(LIBFIDO2_MESSAGE)."
 ifeq ("$(OS)","windows")
 BINARIES=$(BUILDDIR)/tsh
 endif
@@ -216,7 +228,7 @@ $(BUILDDIR)/teleport: ensure-webassets bpf-bytecode rdpclient
 
 .PHONY: $(BUILDDIR)/tsh
 $(BUILDDIR)/tsh:
-	GOOS=$(OS) GOARCH=$(ARCH) $(CGOFLAG_TSH) go build -tags "$(FIPS_TAG)" -o $(BUILDDIR)/tsh $(BUILDFLAGS) ./tool/tsh
+	GOOS=$(OS) GOARCH=$(ARCH) $(CGOFLAG_TSH) go build -tags "$(FIPS_TAG) $(LIBFIDO2_STATIC_TAG)" -o $(BUILDDIR)/tsh $(BUILDFLAGS) ./tool/tsh
 
 .PHONY: $(BUILDDIR)/tbot
 $(BUILDDIR)/tbot:
@@ -847,6 +859,12 @@ docker-binaries: clean
 enter:
 	make -C build.assets enter
 
+# Interactively enters a Docker container (which you can build and run Teleport inside of).
+# Similar to `enter`, but uses the centos7 container.
+.PHONY:enter/centos7
+enter/centos7:
+	make -C build.assets enter/centos7
+
 # grpc generates GRPC stubs from service definitions.
 # This target runs in the buildbox container.
 .PHONY: grpc

build.assets/Dockerfile

@@ -6,7 +6,44 @@
 # runtime requirement for the host to have newer glibc too. For example,
 # teleport built on any newer Ubuntu version will not run on Centos 7 because
 # of this.
-FROM ubuntu:18.04
+
+# Build libfido2 separately for isolation, speed and flexibility.
+FROM buildpack-deps:18.04 AS libfido2
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends cmake && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install libudev-zero.
+# libudev-zero replaces systemd's libudev
+RUN git clone --depth=1 https://github.com/illiliti/libudev-zero.git -b 1.0.1 && \
+    cd libudev-zero && \
+    make install-static
+
+# Install libcbor.
+RUN git clone --depth=1 https://github.com/PJK/libcbor.git -b v0.9.0 && \
+    cd libcbor && \
+    cmake \
+        -DCBOR_CUSTOM_ALLOC=ON \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+        -DWITH_EXAMPLES=OFF . && \
+    make && \
+    make install
+
+# Install libfido2.
+# Depends on libcbor, libssl-dev, zlib1g-dev and libudev.
+RUN git clone --depth=1 https://github.com/Yubico/libfido2.git -b 1.10.0 && \
+    cd libfido2 && \
+    CFLAGS=-pthread cmake \
+        -DBUILD_EXAMPLES=OFF \
+        -DBUILD_MANPAGES=OFF \
+        -DBUILD_TOOLS=OFF \
+        -DCMAKE_BUILD_TYPE=Release . && \
+    make && \
+    make install
+
+FROM ubuntu:18.04 AS buildbox
 
 COPY locale.gen /etc/locale.gen
 COPY profile /etc/profile
@@ -32,7 +69,6 @@ RUN apt-get update -y --fix-missing && \
         ca-certificates \
         clang-10 \
         clang-format-10 \
-        cmake \
         curl \
         default-jre \
         `if [ "$BUILDARCH" = "amd64" ] ; then echo gcc-multilib; fi`  \
@@ -84,29 +120,16 @@ RUN (groupadd ci --gid=$GID -o && useradd ci --uid=$UID --gid=$GID --create-home
 RUN (curl -L https://github.com/coreos/etcd/releases/download/v3.3.9/etcd-v3.3.9-linux-${BUILDARCH}.tar.gz | tar -xz && \
      cp etcd-v3.3.9-linux-${BUILDARCH}/etcd* /bin/)
 
-# Install libcbor.
-RUN git clone --depth=1 https://github.com/PJK/libcbor.git -b v0.9.0 && \
-  cd libcbor && \
-  cmake -DCMAKE_BUILD_TYPE=Release -DCBOR_CUSTOM_ALLOC=ON . && \
-  make && \
-  make install
-
-# Install libfido2.
-# Depends on libcbor, libssl-dev, zlib and libudev-dev.
-RUN git clone --depth=1 https://github.com/Yubico/libfido2.git -b 1.10.0 && \
-  cd libfido2 && \
-  cmake -DCMAKE_BUILD_TYPE=Release . && \
-  make && \
-  make install && \
-  ldconfig
-
 # Install Go.
 ARG GOLANG_VERSION
 RUN mkdir -p /opt && cd /opt && curl https://storage.googleapis.com/golang/$GOLANG_VERSION.linux-${BUILDARCH}.tar.gz | tar xz && \
     mkdir -p /go/src/github.com/gravitational/teleport && \
     chmod a+w /go && \
     chmod a+w /var/lib && \
     chmod a-w /
+ENV GOPATH="/go" \
+    GOROOT="/opt/go" \
+    PATH="$PATH:/opt/go/bin:/go/bin:/go/src/github.com/gravitational/teleport/build"
 
 # Install libbpf
 ARG LIBBPF_VERSION
@@ -115,10 +138,6 @@ RUN mkdir -p /opt && cd /opt && curl -L https://github.com/gravitational/libbpf/
     make && \
     make install
 
-ENV GOPATH="/go" \
-    GOROOT="/opt/go" \
-    PATH="$PATH:/opt/go/bin:/go/bin:/go/src/github.com/gravitational/teleport/build"
-
 # Install addlicense
 RUN go install github.com/google/[email protected]
 
@@ -188,6 +207,21 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --pr
     if [ "$BUILDARCH" = "amd64" ]; then rustup target add i686-unknown-linux-gnu arm-unknown-linux-gnueabihf aarch64-unknown-linux-gnu; fi && \
     cargo install cbindgen
 
+# Do a quick switch back to root and copy/setup libfido2 binaries.
+# Do this last to take better advantage of the multi-stage build.
 USER root
+COPY --from=libfido2 /usr/local/include/ /usr/local/include/
+COPY --from=libfido2 \
+    /usr/local/lib/libcbor.a \
+    /usr/local/lib/libfido2.a \
+    /usr/local/lib/libfido2.so.1.10.0 \
+    /usr/local/lib/libudev.a \
+    /usr/local/lib/
+RUN cd /usr/local/lib && \
+    ln -s libfido2.so.1.10.0 libfido2.so.1 && \
+    ln -s libfido2.so.1 libfido2.so && \
+    ldconfig
+
+USER ci
 VOLUME ["/go/src/github.com/gravitational/teleport"]
 EXPOSE 6600 2379 2380

build.assets/Dockerfile-centos7

@@ -1,4 +1,65 @@
-FROM centos:7
+# Build libfido2 separately for isolation, speed and flexibility.
+FROM centos:7 AS libfido2
+
+RUN yum groupinstall -y 'Development Tools' && \
+    yum install -y epel-release && \
+    yum update -y && \
+    yum install -y \
+        cmake3 \
+        git \
+        libudev-devel \
+        zlib-devel && \
+    yum clean all
+
+# Install libudev-zero.
+# libudev-zero replaces systemd's libudev
+RUN git clone --depth=1 https://github.com/illiliti/libudev-zero.git -b 1.0.1 && \
+    cd libudev-zero && \
+    make install-static
+
+# Instal openssl.
+# Pulled from source because repository versions are too old.
+RUN git clone --depth=1 git://git.openssl.org/openssl.git -b OpenSSL_1_1_1n && \
+    cd openssl && \
+    ./config --release && \
+    make && \
+    make install && \
+# Link for `make build/tsh`.
+    mkdir -p /usr/lib/x86_64-linux-gnu && \
+    ln -s /usr/local/lib64/libcrypto.a /usr/lib/x86_64-linux-gnu/
+
+# Install libcbor.
+RUN git clone --depth=1 https://github.com/PJK/libcbor.git -b v0.9.0 && \
+    cd libcbor && \
+    cmake3 \
+        -DCBOR_CUSTOM_ALLOC=ON \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+        -DWITH_EXAMPLES=OFF . && \
+    make && \
+    make install && \
+# Link for `make build/tsh`.
+    ln -s /usr/local/lib64/libcbor.{a,so} /usr/local/lib/
+
+# Install libfido2.
+# Depends on libcbor, openssl, zlib-devel and libudev.
+# Linked so `make build/tsh` finds the library where it expects it.
+RUN git clone --depth=1 https://github.com/Yubico/libfido2.git -b 1.10.0 && \
+    cd libfido2 && \
+    cmake3 \
+        -DBUILD_EXAMPLES=OFF \
+        -DBUILD_MANPAGES=OFF \
+        -DBUILD_TOOLS=OFF \
+        -DCMAKE_BUILD_TYPE=Release . && \
+    make && \
+    make install && \
+# Link for `make build/tsh`.
+    ln -s /usr/local/lib64/libfido2.{a,so} /usr/local/lib/ && \
+# Update ld.
+    echo /usr/local/lib64 > /etc/ld.so.conf.d/libfido2.conf && \
+    ldconfig
+
+FROM centos:7 AS buildbox
 
 ENV LANGUAGE=en_US.UTF-8 \
     LANG=en_US.UTF-8 \
@@ -13,9 +74,15 @@ ARG GID
 RUN (groupadd ci --gid=$GID -o && useradd ci --uid=$UID --gid=$GID --create-home --shell=/bin/sh && \
      mkdir -p -m0700 /var/lib/teleport && chown -R ci /var/lib/teleport)
 
-# Install dev tools (make, etc) and a Perl package needed to build OpenSSL.
-RUN yum groupinstall -y "Development Tools"
-RUN yum install -y pam-devel net-tools tree git zip libatomic perl-IPC-Cmd && \
+RUN yum groupinstall -y 'Development Tools' && \
+    yum install -y \
+        git \
+        libatomic \
+        net-tools \
+        pam-devel \
+        perl-IPC-Cmd \
+        tree \
+        zip \
     yum clean all
 
 # Install etcd.
@@ -28,6 +95,9 @@ RUN mkdir -p /opt && cd /opt && curl https://storage.googleapis.com/golang/$GOLA
     chmod a+w /go && \
     chmod a+w /var/lib && \
     /opt/go/bin/go version
+ENV GOPATH="/go" \
+    GOROOT="/opt/go" \
+    PATH="/opt/bin:$PATH:/opt/go/bin:/go/bin:/go/src/github.com/gravitational/teleport/build"
 
 # Install PAM module and policies for testing.
 COPY pam/ /opt/pam_teleport/
@@ -52,9 +122,36 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --pr
     rustup component add --toolchain $RUST_VERSION-x86_64-unknown-linux-gnu rustfmt clippy && \
     cargo install cbindgen
 
-ENV GOPATH="/go" \
-    GOROOT="/opt/go" \
-    PATH="/opt/bin:$PATH:/opt/go/bin:/go/bin:/go/src/github.com/gravitational/teleport/build"
+# Do a quick switch back to root and copy/setup libfido2 binaries.
+# Do this last to take better advantage of the multi-stage build.
+USER root
+COPY --from=libfido2 /usr/local/include/ /usr/local/include/
+COPY --from=libfido2 \
+    /usr/local/lib64/libcbor.a \
+    /usr/local/lib64/libcrypto.a \
+    /usr/local/lib64/libcrypto.so.1.1 \
+    /usr/local/lib64/libfido2.a \
+    /usr/local/lib64/libfido2.so.1.10.0 \
+    /usr/local/lib64/libssl.a \
+    /usr/local/lib64/libssl.so.1.1 \
+    /usr/local/lib64/
+COPY --from=libfido2 /usr/local/lib/libudev.a /usr/local/lib/
+RUN cd /usr/local/lib64 && \
+# Re-create usual lib64 links.
+    ln -s libcrypto.so.1.1 libcrypto.so && \
+    ln -s libfido2.so.1.10.0 libfido2.so.1 && \
+    ln -s libfido2.so.1 libfido2.so && \
+    ln -s libssl.so.1.1 libssl.so && \
+# Links for `make build/tsh`.
+    ln -s /usr/local/lib64/libcbor.a \
+        /usr/local/lib64/libfido2.{a,so} \
+        /usr/local/lib/ && \
+    mkdir -p /usr/lib/x86_64-linux-gnu && \
+    ln -s /usr/local/lib64/libcrypto.a /usr/lib/x86_64-linux-gnu/ && \
+# Update ld.
+    echo /usr/local/lib64 > /etc/ld.so.conf.d/libfido2.conf && \
+    ldconfig
 
+USER ci
 VOLUME ["/go/src/github.com/gravitational/teleport"]
 EXPOSE 6600 2379 2380

build.assets/Makefile

@@ -317,6 +317,13 @@ enter: buildbox
 		-e HOME=$(SRCDIR)/build.assets -w $(SRCDIR) $(BUILDBOX) /bin/bash
 
 #
+# Starts shell inside the centos7 container
+#
+.PHONY:enter/centos7
+enter/centos7: buildbox
+	docker run $(DOCKERFLAGS) -ti $(NOROOT) \
+		-e HOME=$(SRCDIR)/build.assets -w $(SRCDIR) $(BUILDBOX_CENTOS7) /bin/bash
+#
 # Create a Teleport package using the build container.
 # Don't use this target directly; call named Makefile targets like release-amd64.
 #

go.mod

@@ -272,7 +272,7 @@ replace (
 	github.com/go-redis/redis/v8 => github.com/gravitational/redis/v8 v8.11.5-0.20220211010318-7af711b76a91
 	github.com/gogo/protobuf => github.com/gravitational/protobuf v1.3.2-0.20201123192827-2b9fcfaffcbf
 	github.com/gravitational/teleport/api => ./api
-	github.com/keys-pub/go-libfido2 => github.com/gravitational/go-libfido2 v1.5.3-0.20220330170708-36815bbb94b7
+	github.com/keys-pub/go-libfido2 => github.com/gravitational/go-libfido2 v1.5.3-0.20220412221142-6da2a1c840d1
 	github.com/russellhaering/gosaml2 => github.com/gravitational/gosaml2 v0.0.0-20220318224559-f06932032ae2
 	github.com/siddontang/go-mysql v1.1.0 => github.com/gravitational/go-mysql v1.1.1-teleport.2
 	github.com/sirupsen/logrus => github.com/gravitational/logrus v1.4.4-0.20210817004754-047e20245621

go.sum

@@ -494,8 +494,8 @@ github.com/gravitational/configure v0.0.0-20180808141939-c3428bd84c23 h1:havbccu
 github.com/gravitational/configure v0.0.0-20180808141939-c3428bd84c23/go.mod h1:XL9nebvlfNVvRzRPWdDcWootcyA0l7THiH/A+W1233g=
 github.com/gravitational/form v0.0.0-20151109031454-c4048f792f70 h1:To76nCJtM3DI0mdq3nGLzXqTV1wNOJByxv01+u9/BxM=
 github.com/gravitational/form v0.0.0-20151109031454-c4048f792f70/go.mod h1:88hFR45MpUd23d2vNWE/dYtesU50jKsbz0I9kH7UaBY=
-github.com/gravitational/go-libfido2 v1.5.3-0.20220330170708-36815bbb94b7 h1:RmlCEGmDJBGaglPQ8ixRH7WBw858PQb0tjxKMFzzUSc=
-github.com/gravitational/go-libfido2 v1.5.3-0.20220330170708-36815bbb94b7/go.mod h1:P0V19qHwJNY0htZwZDe9Ilvs/nokGhdFX7faKFyZ6+U=
+github.com/gravitational/go-libfido2 v1.5.3-0.20220412221142-6da2a1c840d1 h1:5+dqZdP7jXuVsi57DGGxmvBzjRKedsP0WpMLzuNgXF0=
+github.com/gravitational/go-libfido2 v1.5.3-0.20220412221142-6da2a1c840d1/go.mod h1:P0V19qHwJNY0htZwZDe9Ilvs/nokGhdFX7faKFyZ6+U=
 github.com/gravitational/go-mssqldb v0.11.1-0.20220202000043-bec708e9bfd0 h1:DC+S+j/tBs/0MnQCC5j7GWWbMGcR3ca5v75ksAU1LJM=
 github.com/gravitational/go-mssqldb v0.11.1-0.20220202000043-bec708e9bfd0/go.mod h1:iiK0YP1ZeepvmBQk/QpLEhhTNJgfzrpArPY/aFvc9yU=
 github.com/gravitational/go-mysql v1.1.1-teleport.2 h1:XZ36BZ7BgslA5ZCyCHjpc1wilFITThIH7cLcbLWKWzM=