diff --git a/Makefile b/Makefile index a29f72dd6d115..4da16938271af 100644 --- a/Makefile +++ b/Makefile @@ -1118,6 +1118,9 @@ enter-root: enter/centos7: make -C build.assets enter/centos7 +.PHONY:enter/grpcbox +enter/grpcbox: + make -C build.assets enter/grpcbox BUF := buf diff --git a/build.assets/Dockerfile b/build.assets/Dockerfile index e5eae086e3b52..b01986f6c0612 100644 --- a/build.assets/Dockerfile +++ b/build.assets/Dockerfile @@ -272,19 +272,22 @@ RUN helm plugin install https://github.com/vbehar/helm3-unittest && \ HELM_PLUGINS=/home/ci/.local/share/helm/plugins helm plugin list # Install JS gRPC tools. -RUN (npm install --global grpc_tools_node_protoc_ts@5.0.1 grpc-tools@1.12.4) - -# Install protobuf and grpc build tools. -ARG PROTOC_VER -ARG GOGO_PROTO_TAG - -RUN (export PROTOC_TARBALL=protoc-${PROTOC_VER}-linux-$(if [ "$BUILDARCH" = "amd64" ]; then echo "x86_64"; else echo "aarch_64"; fi).zip && \ - curl -fsSL -o /tmp/${PROTOC_TARBALL} https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VER}/${PROTOC_TARBALL} && \ - cd /tmp && unzip /tmp/${PROTOC_TARBALL} -d /usr/local && \ - chmod -R a+r /usr/local/include/google/protobuf && \ - chmod -R a+xr /usr/local/bin/protoc && \ - rm /tmp/${PROTOC_TARBALL}) -RUN go install github.com/gogo/protobuf/protoc-gen-gogofast@${GOGO_PROTO_TAG} +ARG NODE_GRPC_TOOLS_VERSION # eg, "1.12.4" +ARG NODE_PROTOC_TS_VERSION # eg, "5.0.1" +RUN npm install --global "grpc-tools@$NODE_GRPC_TOOLS_VERSION" "grpc_tools_node_protoc_ts@$NODE_PROTOC_TS_VERSION" + +# Install protoc. +ARG PROTOC_VER # eg, "3.20.2" +RUN VERSION="$PROTOC_VER" && \ + PB_REL='https://github.com/protocolbuffers/protobuf/releases' && \ + PB_FILE="$(mktemp protoc-XXXXXX.zip)" && \ + curl -fsSL -o "$PB_FILE" "$PB_REL/download/v$VERSION/protoc-$VERSION-linux-$(if [ "$BUILDARCH" = "amd64" ]; then echo "x86_64"; else echo "aarch_64"; fi).zip" && \ + unzip "$PB_FILE" -d /usr/local && \ + rm -f "$PB_FILE" + +# Install protoc-gen-gogofast. +ARG GOGO_PROTO_TAG # eg, "v1.3.2" +RUN go install "github.com/gogo/protobuf/protoc-gen-gogofast@$GOGO_PROTO_TAG" # Install addlicense. RUN go install github.com/google/addlicense@v1.0.0 @@ -297,12 +300,11 @@ RUN curl -fsSL "https://raw.githubusercontent.com/golangci/golangci-lint/v1.52.2 sh -s -- -b "$(go env GOPATH)/bin" # Install Buf. -RUN BIN="/usr/local/bin" && \ - VERSION="1.19.0" && \ - curl -fsSL \ - "https://github.com/bufbuild/buf/releases/download/v${VERSION}/buf-$(uname -s)-$(uname -m)" \ - -o "${BIN}/buf" && \ - chmod +x "${BIN}/buf" +ARG BUF_VERSION # eg, "1.19.0" +RUN BIN='/usr/local/bin' && \ + VERSION="$BUF_VERSION" && \ + curl -fsSL "https://github.com/bufbuild/buf/releases/download/v$VERSION/buf-$(uname -s)-$(uname -m)" -o "${BIN}/buf" && \ + chmod +x "$BIN/buf" # Copy BPF libraries. ARG LIBBPF_VERSION diff --git a/build.assets/Dockerfile-grpcbox b/build.assets/Dockerfile-grpcbox new file mode 100644 index 0000000000000..6611235862e71 --- /dev/null +++ b/build.assets/Dockerfile-grpcbox @@ -0,0 +1,45 @@ +FROM golang:1.20 + +# Image layers go from less likely to most likely to change. +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + npm \ + unzip \ + && \ + rm -rf /var/lib/apt/lists/* + +# protoc-gen-gogofast +# Keep in sync with api/proto/buf.yaml (and buf.lock) +ARG GOGO_PROTO_TAG # eg, "v1.3.2" +RUN go install "github.com/gogo/protobuf/protoc-gen-gogofast@$GOGO_PROTO_TAG" + +# protoc-gen-js and protoc-gen-ts +ARG NODE_GRPC_TOOLS_VERSION # eg, "1.12.4" +ARG NODE_PROTOC_TS_VERSION # eg, "5.0.1" +RUN npm install --global "grpc-tools@$NODE_GRPC_TOOLS_VERSION" "grpc_tools_node_protoc_ts@$NODE_PROTOC_TS_VERSION" + +# protoc +ARG PROTOC_VER # eg, "3.20.2" +RUN VERSION="$PROTOC_VER" && \ + PB_REL='https://github.com/protocolbuffers/protobuf/releases' && \ + PB_FILE="$(mktemp protoc-XXXXXX.zip)" && \ + ARCH="$(if [ "$(uname -m)" = aarch64 ]; then echo aarch_64; else uname -m; fi)" && \ + curl -fsSL -o "$PB_FILE" "$PB_REL/download/v$VERSION/protoc-$VERSION-linux-$ARCH.zip" && \ + unzip "$PB_FILE" -d /usr/local && \ + rm -f "$PB_FILE" + +# buf +ARG BUF_VERSION # eg, "1.19.0" +RUN BIN='/usr/local/bin' && \ + VERSION="$BUF_VERSION" && \ + curl -fsSL "https://github.com/bufbuild/buf/releases/download/v$VERSION/buf-$(uname -s)-$(uname -m)" -o "${BIN}/buf" && \ + chmod +x "$BIN/buf" + +# Pre-install go-runned binaries. +# This is meant to be the only step that changes depending on the Teleport +# branch. +COPY go.mod go.sum /teleport-module/ +RUN cd /teleport-module; \ + go install github.com/bufbuild/connect-go/cmd/protoc-gen-connect-go && \ + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc && \ + go install google.golang.org/protobuf/cmd/protoc-gen-go diff --git a/build.assets/Makefile b/build.assets/Makefile index 8e0554f936553..b87b9de034c66 100644 --- a/build.assets/Makefile +++ b/build.assets/Makefile @@ -28,6 +28,14 @@ RUST_VERSION ?= 1.68.0 LIBBPF_VERSION ?= 1.0.1 LIBPCSCLITE_VERSION ?= 1.9.9-teleport +# Protogen related versions. +BUF_VERSION ?= 1.19.0 +# Keep in sync with api/proto/buf.yaml (and buf.lock) +GOGO_PROTO_TAG ?= v1.3.2 +NODE_GRPC_TOOLS_VERSION ?= 1.12.4 +NODE_PROTOC_TS_VERSION ?= 5.0.1 +PROTOC_VER ?= 3.20.3 + UID := $$(id -u) GID := $$(id -g) @@ -38,12 +46,9 @@ RUNTIME_ARCH_arm64 := arm64 RUNTIME_ARCH_aarch64 := arm64 RUNTIME_ARCH := $(RUNTIME_ARCH_$(HOST_ARCH)) -PROTOC_VER ?= 3.20.3 -# Keep in sync with api/proto/buf.yaml (and buf.lock) -GOGO_PROTO_TAG ?= v1.3.2 - # BUILDBOX_VERSION, BUILDBOX and BUILDBOX_variant variables are included include images.mk +include grpcbox.mk # These variables are used to dynamically change the name of the buildbox Docker image used by the 'release' # target. The other solution was to remove the 'buildbox' dependency from the 'release' target, but this would @@ -131,9 +136,12 @@ buildbox: --build-arg GOLANG_VERSION=$(GOLANG_VERSION) \ --build-arg RUST_VERSION=$(RUST_VERSION) \ --build-arg NODE_VERSION=$(NODE_VERSION) \ - --build-arg PROTOC_VER=$(PROTOC_VER) \ - --build-arg GOGO_PROTO_TAG=$(GOGO_PROTO_TAG) \ --build-arg LIBBPF_VERSION=$(LIBBPF_VERSION) \ + --build-arg BUF_VERSION=$(BUF_VERSION) \ + --build-arg GOGO_PROTO_TAG=$(GOGO_PROTO_TAG) \ + --build-arg NODE_GRPC_TOOLS_VERSION=$(NODE_GRPC_TOOLS_VERSION) \ + --build-arg NODE_PROTOC_TS_VERSION=$(NODE_PROTOC_TS_VERSION) \ + --build-arg PROTOC_VER=$(PROTOC_VER) \ --cache-from $(BUILDBOX) \ --tag $(BUILDBOX) . ; \ fi @@ -170,7 +178,6 @@ buildbox-centos7: --build-arg NODE_VERSION=$(NODE_VERSION) \ --build-arg RUST_VERSION=$(RUST_VERSION) \ --build-arg DEVTOOLSET=$(DEVTOOLSET) \ - --build-arg PROTOC_VER=$(PROTOC_VER) \ --build-arg LIBBPF_VERSION=$(LIBBPF_VERSION) \ --build-arg LIBPCSCLITE_VERSION=$(LIBPCSCLITE_VERSION) \ --cache-from $(BUILDBOX_CENTOS7) \ @@ -242,17 +249,13 @@ ui: buildbox # grpc generates GRPC stubs from inside the buildbox .PHONY: grpc -grpc: buildbox - docker run \ - $(DOCKERFLAGS) -u $(UID):$(GID) -t $(BUILDBOX) \ - make -C /go/src/github.com/gravitational/teleport grpc/host +grpc: grpcbox + $(GRPCBOX_RUN) make grpc/host # protos-up-to-date checks if GRPC stubs are up to date from inside the buildbox .PHONY: protos-up-to-date -protos-up-to-date: buildbox - docker run \ - $(DOCKERFLAGS) -t $(BUILDBOX) \ - make -C /go/src/github.com/gravitational/teleport protos-up-to-date/host +protos-up-to-date: grpcbox + $(GRPCBOX_RUN) make protos-up-to-date/host # fix-imports runs GCI to sort and re-order Go imports in a deterministic way. .PHONY: fix-imports @@ -378,6 +381,13 @@ enter-root/centos7: buildbox-centos7 docker run $(DOCKERFLAGS) -ti \ -e HOME=$(SRCDIR)/build.assets -w $(SRCDIR) $(BUILDBOX_CENTOS7) /bin/bash +# +# Starts a shell inside the grpcbox. +# +.PHONY: enter/grpcbox +enter/grpcbox: grpcbox + $(GRPCBOX_RUN) + # # Create a Teleport package using the build container. # Don't use this target directly; call named Makefile targets like release-amd64. diff --git a/build.assets/grpcbox.mk b/build.assets/grpcbox.mk new file mode 100644 index 0000000000000..b489041210532 --- /dev/null +++ b/build.assets/grpcbox.mk @@ -0,0 +1,33 @@ +# Makefile for grpcbox targets and variables. +# +# The grpcbox is a leaner, meaner, faster buildbox meant exclusively for +# codegen. +# It is not guaranteed to have tooling parity with the buildbox. +# +# See Dockerfile-grpcbox. + +GRPCBOX_BASE_NAME ?= teleport-grpcbox + +ifeq ($(BUILDBOX_VERSION), "") +GRPCBOX ?= $(GRPCBOX_BASE_NAME) +else +GRPCBOX ?= $(GRPCBOX_BASE_NAME):$(BUILDBOX_VERSION) +endif + +# GRPCBOX_RUN has the necessary invocation to run a command inside the grpcbox. +# Use this variable to run it from other Makefiles. +GRPCBOX_RUN := docker run -it --rm -v "$$(pwd)/../:/workdir" -w /workdir $(GRPCBOX) + +# grpcbox builds a codegen-focused buildbox. +# It's leaner, meaner, faster and not supposed to compile code. +.PHONY: grpcbox +grpcbox: + DOCKER_BUILDKIT=1 docker build \ + --build-arg BUF_VERSION=$(BUF_VERSION) \ + --build-arg GOGO_PROTO_TAG=$(GOGO_PROTO_TAG) \ + --build-arg NODE_GRPC_TOOLS_VERSION=$(NODE_GRPC_TOOLS_VERSION) \ + --build-arg NODE_PROTOC_TS_VERSION=$(NODE_PROTOC_TS_VERSION) \ + --build-arg PROTOC_VER=$(PROTOC_VER) \ + -f Dockerfile-grpcbox \ + -t "$(GRPCBOX)" \ + ../