Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Draft] Cross-compilation MVP #36066

Draft
wants to merge 20 commits into
base: master
Choose a base branch
from
Draft

[Draft] Cross-compilation MVP #36066

Show file tree
Hide file tree
Changes from 15 commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
0abcc75
Experimental ct-ng build
jakule Dec 26, 2023
de0e34c
Add new AMD64 configuration file for experimental ct-ng build
jakule Dec 26, 2023
ea8f7bf
Update build script and configurations for cross-compilation
jakule Dec 27, 2023
86d8fef
Update build-all.sh for i686
jakule Dec 27, 2023
cc4de2f
Update arm build configurations
jakule Dec 27, 2023
bc47563
Add arm build configurations
jakule Dec 27, 2023
56cd062
Add Dockerfile for crosstool-ng and update build script
jakule Dec 27, 2023
b98d2db
Move build script and update Dockerfile build settings
jakule Dec 27, 2023
cc794b7
Refactor build script and update multi-arch Docker build settings
jakule Dec 27, 2023
f53f677
Refactor build scripts for multi-architecture Docker builds
jakule Dec 27, 2023
1c574ad
Enhance cross-compilation handling in build scripts
jakule Dec 27, 2023
fe311cd
Optimize build scripts for cross-compilation and Docker building
jakule Dec 27, 2023
4f9200e
Refine build process for arm architecture
jakule Dec 27, 2023
691e0a6
Improve build script and update data types in client.go
jakule Dec 28, 2023
4aa3cab
Add support for MacOS M1
jakule Dec 28, 2023
7b78274
Merge remote-tracking branch 'origin/master' into jakule/cross-compil…
jakule Jul 15, 2024
2c793b4
Fix incorrect merge
jakule Jul 15, 2024
6250d29
Update build configurations and toolchain settings for arm64
jakule Jul 16, 2024
b41db8b
Lower required glibc/bintuils arm64 version
jakule Jul 16, 2024
1468a97
Set glibc to 2.17
jakule Jul 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 8 additions & 21 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ endif

# PAM support will only be built into Teleport if headers exist at build time.
PAM_MESSAGE := without-PAM-support
ifneq ("$(wildcard /usr/include/security/pam_appl.h)","")
ifneq ("$(wildcard ${SYSROOT}/include/security/pam_appl.h)","")
PAM_TAG := pam
PAM_MESSAGE := with-PAM-support
else
Expand Down Expand Up @@ -101,6 +101,8 @@ CARGO_TARGET_darwin_amd64 := x86_64-apple-darwin
CARGO_TARGET_darwin_arm64 := aarch64-apple-darwin
CARGO_TARGET_linux_arm64 := aarch64-unknown-linux-gnu
CARGO_TARGET_linux_amd64 := x86_64-unknown-linux-gnu
CARGO_TARGET_linux_386 := i686-unknown-linux-gnu
CARGO_TARGET_linux_arm := armv7-unknown-linux-gnueabi

CARGO_TARGET := --target=${CARGO_TARGET_${OS}_${ARCH}}

Expand All @@ -115,17 +117,12 @@ ifeq ($(RDPCLIENT_SKIP_BUILD),0)
ifneq ($(CHECK_RUST),)
ifneq ($(CHECK_CARGO),)

# Do not build RDP client on ARM or 386.
ifneq ("$(ARCH)","arm")
ifneq ("$(ARCH)","386")
with_rdpclient := yes
RDPCLIENT_MESSAGE := with-Windows-RDP-client
RDPCLIENT_TAG := desktop_access_rdp
endif
endif

endif
endif
endif

# Set C_ARCH for building libfido2 and dependencies. ARCH is the Go
Expand Down Expand Up @@ -232,22 +229,12 @@ TEST_LOG_DIR = ${abspath ./test-logs}

# Set CGOFLAG and BUILDFLAGS as needed for the OS/ARCH.
ifeq ("$(OS)","linux")
# True if $ARCH == amd64 || $ARCH == arm64
ifeq ("$(ARCH)","arm64")
ifeq ($(IS_NATIVE_BUILD),"no")
CGOFLAG += CC=aarch64-linux-gnu-gcc
endif
else ifeq ("$(ARCH)","arm")
ifeq ("$(ARCH)","arm")
CGOFLAG = CGO_ENABLED=1

# ARM builds need to specify the correct C compiler
ifeq ($(IS_NATIVE_BUILD),"no")
CC=arm-linux-gnueabihf-gcc
endif

# Add -debugtramp=2 to work around 24 bit CALL/JMP instruction offset.
BUILDFLAGS = $(ADDFLAGS) -ldflags '-w -s -debugtramp=2 $(KUBECTL_SETVERSION)' -trimpath -buildmode=pie
endif
BUILDFLAGS = $(ADDFLAGS) -ldflags '-w -s -debugtramp=2 $(KUBECTL_SETVERSION)' -trimpath
endif # ARCH == arm
endif # OS == linux

# Windows requires extra parameters to cross-compile with CGO.
Expand Down Expand Up @@ -321,12 +308,12 @@ $(RS_BPF_BUILDDIR):

# Build BPF code
$(ER_BPF_BUILDDIR)/%.bpf.o: bpf/enhancedrecording/%.bpf.c $(wildcard bpf/*.h) | $(ER_BPF_BUILDDIR)
$(CLANG) -g -O2 -target bpf -D__TARGET_ARCH_$(KERNEL_ARCH) -I/usr/libbpf-${LIBBPF_VER}/include $(INCLUDES) $(CLANG_BPF_SYS_INCLUDES) -c $(filter %.c,$^) -o $@
$(CLANG) -g -O2 -target bpf -D__TARGET_ARCH_$(KERNEL_ARCH) -I${SYSROOT}/usr/include/ $(INCLUDES) $(CLANG_BPF_SYS_INCLUDES) -c $(filter %.c,$^) -o $@
$(LLVM_STRIP) -g $@ # strip useless DWARF info

# Build BPF code
$(RS_BPF_BUILDDIR)/%.bpf.o: bpf/restrictedsession/%.bpf.c $(wildcard bpf/*.h) | $(RS_BPF_BUILDDIR)
$(CLANG) -g -O2 -target bpf -D__TARGET_ARCH_$(KERNEL_ARCH) -I/usr/libbpf-${LIBBPF_VER}/include $(INCLUDES) $(CLANG_BPF_SYS_INCLUDES) -c $(filter %.c,$^) -o $@
$(CLANG) -g -O2 -target bpf -D__TARGET_ARCH_$(KERNEL_ARCH) -I${SYSROOT}/usr/include $(INCLUDES) $(CLANG_BPF_SYS_INCLUDES) -c $(filter %.c,$^) -o $@
$(LLVM_STRIP) -g $@ # strip useless DWARF info

.PHONY: bpf-rs-bytecode
Expand Down
45 changes: 45 additions & 0 deletions build.assets/Dockerfile-build
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
FROM ubuntu:22.04
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add some docs/comments on what this dockerfile and the other do? We have so many under build.assets/** that I don't even know what they all do anymore.


# Install dependencies
RUN apt-get update && apt-get install -y \

Check warning on line 4 in build.assets/Dockerfile-build

View check run for this annotation

Orca Security (US) / Orca Security - Infrastructure as Code

[INFO] APT-GET Not Avoiding Additional Packages 

Details:
Check if any apt-get installs don't use '--no-install-recommends' flag to avoid
installing additional packages.

Recommendation:
'RUN apt-get update && apt-get install -y autoconf automake autopoint bison flex
libtool sed w3m xsltproc xz-utils cmake gettext git make pkg-config wget curl
clang-12 && rm -rf /var/lib/apt/lists/*' uses '--no-install-recommends' flag to
avoid installing additional packages
autoconf \
automake \
autopoint \
bison \
flex \
libtool \
sed \
w3m \
xsltproc \
xz-utils \
cmake \
gettext \
git \
make \
pkg-config \
wget \
curl \
clang-12 \
&& rm -rf /var/lib/apt/lists/*

# Install Go
ARG GOLANG_VERSION
RUN mkdir -p /opt && cd /opt && curl -fsSL https://storage.googleapis.com/golang/${GOLANG_VERSION}.linux-amd64.tar.gz | tar xz && \
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Version is pinned, but we could use a checksum here to validate the tarball is what we expect. This isn't super important as we have a higher level of trust for Google and golang distribution.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is PoC, and this code has a lot of bad practices. Versions are not pinned, the repositories are GH mirrors instead of the official. Code is built from master instead of a pinned version. Build options are not set etc.
I wanted to show an alternative approach to what we are using today. I fully agree that the production version should have all those concerns addressed.

/opt/go/bin/go version
ENV GOPATH="/home/ubuntu/go" \
GOROOT="/opt/go" \
PATH="/opt/go/bin:$PATH"

# Create a non-root user with id 1000 - ubuntu
RUN useradd -m -u 1000 ubuntu
USER ubuntu
WORKDIR /home/ubuntu

# Install Rust
ARG RUST_VERSION
ENV PATH=/home/ubuntu/.cargo/bin:$PATH
RUN curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain $RUST_VERSION && \
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd recommend we vendor the shell script and checksum the resulting binary that it fetches. This is probably the worst offender IMO.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I copied this from our "production" code 🙈
https://github.com/gravitational/teleport/blob/master/build.assets/Dockerfile-centos7#L252

rustup --version && \
cargo --version && \
rustc --version && \
rustup target add x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu i686-unknown-linux-gnu armv7-unknown-linux-gnueabi
57 changes: 57 additions & 0 deletions build.assets/Dockerfile-ct-ng
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
# Base on https://github.com/crosstool-ng/crosstool-ng/blob/8825cfc2abf696395bd27bd0e7cea3653004280b/testing/docker/ubuntu22.04/Dockerfile
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Probably need to check with the open source group, but if this dockerfile is based on that link then we may need to include a GPL license header with copyright listed as crosstool-ng

FROM ubuntu:22.04 as ct-ng

ARG CTNG_UID=1000
ARG CTNG_GID=1000

RUN groupadd -g $CTNG_GID ctng

Check warning on line 7 in build.assets/Dockerfile-ct-ng

View check run for this annotation

Orca Security (US) / Orca Security - Infrastructure as Code

[LOW] Multiple RUN, ADD, COPY, Instructions Listed 

Details:
Multiple commands (RUN, COPY, ADD) should be grouped in order to reduce the
number of layers.

Recommendation:
There isn´t any RUN instruction that could be grouped
RUN useradd -d /home/ctng -m -g $CTNG_GID -u $CTNG_UID -s /bin/bash ctng

# Non-interactive configuration of tzdata
ENV DEBIAN_FRONTEND noninteractive
ENV DEBCONF_NONINTERACTIVE_SEEN true
RUN { echo 'tzdata tzdata/Areas select Etc'; echo 'tzdata tzdata/Zones/Etc select UTC'; } | debconf-set-selections

RUN apt-get update

Check warning on line 15 in build.assets/Dockerfile-ct-ng

View check run for this annotation

Orca Security (US) / Orca Security - Infrastructure as Code

[INFO] Update Instruction Alone 

Details:
Instruction 'RUN <package-manager> update' should always be followed by
'<package-manager> install' in the same RUN statement

Recommendation:
Instruction 'RUN <package-manager> update' should be followed by 'RUN
<package-manager> install'
RUN apt-get install -y gcc g++ gperf bison flex texinfo help2man make libncurses5-dev \

Check warning on line 16 in build.assets/Dockerfile-ct-ng

View check run for this annotation

Orca Security (US) / Orca Security - Infrastructure as Code

[INFO] APT-GET Not Avoiding Additional Packages 

Details:
Check if any apt-get installs don't use '--no-install-recommends' flag to avoid
installing additional packages.

Recommendation:
'RUN apt-get install -y gcc g++ gperf bison flex texinfo help2man make
libncurses5-dev python3-dev autoconf automake libtool libtool-bin gawk wget bzip2
xz-utils unzip patch libstdc++6 rsync git meson ninja-build' uses
'--no-install-recommends' flag to avoid installing additional packages

Check warning on line 16 in build.assets/Dockerfile-ct-ng

View check run for this annotation

Orca Security (US) / Orca Security - Infrastructure as Code

[INFO] Apt Get Install Lists Were Not Deleted 

Details:
After using apt-get install, it is needed to delete apt-get lists

Recommendation:
After using apt-get install, the apt-get lists should be deleted
python3-dev autoconf automake libtool libtool-bin gawk wget bzip2 xz-utils unzip \
patch libstdc++6 rsync git meson ninja-build

USER ctng
WORKDIR /home/ctng

RUN wget https://github.com/crosstool-ng/crosstool-ng/releases/download/crosstool-ng-1.26.0/crosstool-ng-1.26.0.tar.bz2 && \

Check warning on line 23 in build.assets/Dockerfile-ct-ng

View check run for this annotation

Orca Security (US) / Orca Security - Infrastructure as Code

[MEDIUM] RUN Instruction Using 'cd' Instead of WORKDIR 

Details:
When using RUN command 'cd' should only be used for full path. For relative path
make use of WORKDIR command instead.

Recommendation:
Using WORKDIR to change directory
Copy link
Contributor

@wadells wadells Dec 27, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd recommend we verify the checksum of this tarball. This isn't as important because we have a pinned release (can be overwritten afaik) and can probably trust GitHub/Microsoft to not get hacked. We're more protecting against the crosstools-ng org/user being compromised.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

#36066 (comment)

tar xf crosstool-ng-1.26.0.tar.bz2 && \
cd crosstool-ng-1.26.0 && \
./bootstrap && \
./configure --prefix=/home/ctng/.local/ct-ng && \
make -j$(nproc) && \
make install && \
cd .. && \
rm -rf crosstool-ng-1.26.0 crosstool-ng-1.26.0.tar.bz2

ENV PATH="/home/ctng/.local/ct-ng/bin:$PATH"

RUN mkdir -p /home/ctng/build/amd64 && \
mkdir -p /home/ctng/build/i686 && \
mkdir -p /home/ctng/build/arm64 && \
mkdir -p /home/ctng/build/arm && \
mkdir -p /home/ctng/src
COPY ./ct-ng-configs/amd64.config /home/ctng/build/amd64/.config
COPY ./ct-ng-configs/i686.config /home/ctng/build/i686/.config
COPY ./ct-ng-configs/arm64.config /home/ctng/build/arm64/.config
COPY ./ct-ng-configs/arm.config /home/ctng/build/arm/.config

WORKDIR /home/ctng/build

#
#FROM ubuntu:22.04
#
## Create a non-root user with id 1000 - ubuntu
#RUN useradd -m -u 1000 ubuntu
#USER ubuntu
#WORKDIR /home/ubuntu
#
#COPY --from=ct-ng /home/ctng/x-tools /home/ubuntu/x-tools
Comment on lines +47 to +55
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This looks like a nice 2-phase build. I'm curious as to why you've commented it out?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Forgot to remove. The idea is simple:

  1. We build a simple container with ct-ng
  2. We use that container to build the cross-compiler
  3. We build container Implement a functional prototype #2 with minimal dependencies and use it to build Teleport.

For some reason, when I tried to build the cross-compiler as a part of docker build, I was getting a weird build failure that I was not able to reproduce anywhere else. This part included the cross-compiler in another stage, but because it was not working, I commented it out and switched to building cross-compiler using docker run instead of docker build.



37 changes: 28 additions & 9 deletions build.assets/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -428,22 +428,38 @@ enter/arm: buildbox-arm
# These are aliases used to make build commands uniform.
# #############################################################################

.PHONY: teleport-build-docker
teleport-build-docker:
docker build -f Dockerfile-build --build-arg GOLANG_VERSION=$(GOLANG_VERSION) --build-arg RUST_VERSION=$(RUST_VERSION) -t teleport-multibuild .

# AMD64 builds are done on CentOS 7 build boxes for broader glibc compatibility.
.PHONY: release-amd64
release-amd64:
$(MAKE) release-centos7 ARCH=amd64 FIDO2=yes PIV=yes
release-amd64: teleport-build-docker
docker run -it --rm -v toolchain:/home/ubuntu/x-tools \
-v 3rdparty:/home/ubuntu/teleport/build.assets/3rdparty \
-v $(shell pwd)/build-arch.sh:/home/ubuntu/teleport/build.assets/build-arch.sh:ro \
-v $(shell pwd)/..:/home/ubuntu/teleport \
-w /home/ubuntu/teleport -u 1000:1000 teleport-multibuild bash -c "cd build.assets && ./build-arch.sh amd64"

.PHONY: release-amd64-fips
release-amd64-fips:
$(MAKE) release-centos7-fips ARCH=amd64 FIDO2=yes PIV=yes FIPS=yes

.PHONY: release-386
release-386:
$(MAKE) release ARCH=386
release-386: teleport-build-docker
docker run -it --rm -v toolchain:/home/ubuntu/x-tools \
-v 3rdparty:/home/ubuntu/teleport/build.assets/3rdparty \
-v $(shell pwd)/build-arch.sh:/home/ubuntu/teleport/build.assets/build-arch.sh:ro \
-v $(shell pwd)/..:/home/ubuntu/teleport \
-w /home/ubuntu/teleport -u 1000:1000 teleport-multibuild bash -c "cd build.assets && ./build-arch.sh 386"

.PHONY: release-arm64
release-arm64:
$(MAKE) release-centos7 ARCH=arm64 FIDO2=yes PIV=yes
release-arm64: teleport-build-docker
docker run -it --rm -v toolchain:/home/ubuntu/x-tools \
-v 3rdparty:/home/ubuntu/teleport/build.assets/3rdparty \
-v $(shell pwd)/build-arch.sh:/home/ubuntu/teleport/build.assets/build-arch.sh:ro \
-v $(shell pwd)/..:/home/ubuntu/teleport \
-w /home/ubuntu/teleport -u 1000:1000 teleport-multibuild bash -c "cd build.assets && ./build-arch.sh arm64"

.PHONY: release-arm64-fips
release-arm64-fips:
Expand All @@ -453,9 +469,12 @@ release-arm64-fips:
# so that we don't need to install node.js and rust on the buildbox-arm container.
# Therefore we set RELEASE_TARGET to release-unix-preserving-webassets.
.PHONY: release-arm
release-arm: webassets
$(MAKE) release RELEASE_TARGET=release-unix-preserving-webassets ARCH=arm BUILDBOX=$(BUILDBOX_ARM)

release-arm: teleport-build-docker
docker run -it --rm -v toolchain:/home/ubuntu/x-tools \
-v 3rdparty:/home/ubuntu/teleport/build.assets/3rdparty \
-v $(shell pwd)/build-arch.sh:/home/ubuntu/teleport/build.assets/build-arch.sh:ro \
-v $(shell pwd)/..:/home/ubuntu/teleport \
-w /home/ubuntu/teleport -u 1000:1000 teleport-multibuild bash -c "cd build.assets && ./build-arch.sh arm"
# Compatibility targets for when we had separate Ubuntu and CentOS 7 targets for AMD64
# TODO(camscale): Remove these when drone no longer calls them.
.PHONY: release-amd64-centos7
Expand Down
21 changes: 21 additions & 0 deletions build.assets/build-all.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
#!/bin/bash

set -e

mkdir -p toolchains
DOCKER_BUILDKIT=0 BUILDKIT_PROGRESS=plain docker build -t teleport-builder-base -f Dockerfile-ct-ng .

docker volume create toolchain

docker run --rm -v toolchain:/toolchain busybox \
/bin/sh -c 'touch /toolchain/.initialized && chown -R 1000:1000 /toolchain'

docker volume create 3rdparty

docker run --rm -v 3rdparty:/3rdparty busybox \
/bin/sh -c 'chown -R 1000:1000 /3rdparty'

docker run -v toolchain:/home/ctng/x-tools --rm docker.io/library/teleport-builder-base bash -c "cd amd64 && ct-ng build"
docker run -v toolchain:/home/ctng/x-tools --rm docker.io/library/teleport-builder-base bash -c "cd i686 && ct-ng build"
docker run -v toolchain:/home/ctng/x-tools --rm docker.io/library/teleport-builder-base bash -c "cd arm64 && ct-ng build"
docker run -v toolchain:/home/ctng/x-tools --rm docker.io/library/teleport-builder-base bash -c "cd arm && ct-ng build"
Loading
Loading