diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 393b1028..68af7b33 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -42,11 +42,11 @@ steps: - "false" - "true" - - label: ":linux: Staging / Ubuntu ARM - Makefile.debian9 - fips: {{matrix.fips}}" + - label: ":linux: Staging / Ubuntu ARM - {{matrix.makefile}} - fips: {{matrix.fips}}" key: "build-ubuntu-arm" command: - - ".buildkite/scripts/build.sh Makefile.debian9" - - ".buildkite/scripts/publish.sh Makefile.debian9" + - ".buildkite/scripts/build.sh {{matrix.makefile}}" + - ".buildkite/scripts/publish.sh {{matrix.makefile}}" env: REPOSITORY: "${STAGING_IMAGE}" FIPS: "{{matrix.fips}}" @@ -60,6 +60,11 @@ steps: instanceType: "t4g.large" matrix: setup: + makefile: + - "Makefile.debian9" + - "Makefile.debian10" + - "Makefile.debian11" + - "Makefile.debian12" fips: - "false" - "true" @@ -95,11 +100,11 @@ steps: - github_commit_status: context: "Release / Ubuntu X86_64" - - label: ":linux: Release / Ubuntu ARM - Makefile.debian9 - fips: {{matrix.fips}}" + - label: ":linux: Release / Ubuntu ARM - {{matrix.makefile}} - fips: {{matrix.fips}}" key: "release-ubuntu-arm" command: - - ".buildkite/scripts/build.sh Makefile.debian9" - - ".buildkite/scripts/publish.sh Makefile.debian9" + - ".buildkite/scripts/build.sh {{matrix.makefile}}" + - ".buildkite/scripts/publish.sh {{matrix.makefile}}" env: FIPS: "{{matrix.fips}}" if: build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+$$/ @@ -109,6 +114,11 @@ steps: instanceType: "t4g.large" matrix: setup: + makefile: + - "Makefile.debian9" + - "Makefile.debian10" + - "Makefile.debian11" + - "Makefile.debian12" fips: - "false" - "true" diff --git a/Makefile.common b/Makefile.common index 5e0dd98b..de2b9e6d 100644 --- a/Makefile.common +++ b/Makefile.common @@ -9,13 +9,6 @@ SUFFIX_NPCAP_VERSION := -npcap-$(NPCAP_VERSION) NPCAP_REPOSITORY := docker.elastic.co/observability-ci GS_BUCKET_PATH ?= ingest-buildkite-ci - -ifeq ($(BUILDX),1) -ifeq ($(shell test $(DEBIAN_VERSION) -ge 10; echo $$?),0) -DOCKER_MULTIARCH := 1 -endif -endif - # Requires login at google storage. copy-npcap: ifeq ($(CI),true) diff --git a/go/Makefile.debian10 b/go/Makefile.debian10 index 0ae5d8f4..82ae73b7 100644 --- a/go/Makefile.debian10 +++ b/go/Makefile.debian10 @@ -1,4 +1,5 @@ -IMAGES ?= base main darwin arm armhf darwin-arm64 npcap +IMAGES := base main darwin armhf npcap +ARM_IMAGES := base-arm darwin-arm64 DEBIAN_VERSION := 10 TAG_EXTENSION := -debian10 @@ -7,8 +8,14 @@ export DEBIAN_VERSION TAG_EXTENSION build: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) build || exit 1;) +build-arm: + @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) build-arm || exit 1;) + # Requires login at https://docker.elastic.co:7000/. push: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) push || exit 1;) -.PHONY: build push +push-arm: + @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) push-arm || exit 1;) + +.PHONY: build build-arm push push-arm diff --git a/go/Makefile.debian11 b/go/Makefile.debian11 index 24da642c..f7631965 100644 --- a/go/Makefile.debian11 +++ b/go/Makefile.debian11 @@ -1,4 +1,5 @@ -IMAGES := base main darwin arm armhf darwin-arm64 npcap +IMAGES := base main darwin armhf npcap +ARM_IMAGES := base-arm darwin-arm64 DEBIAN_VERSION := 11 TAG_EXTENSION := -debian11 @@ -7,8 +8,14 @@ export DEBIAN_VERSION TAG_EXTENSION build: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) build || exit 1;) +build-arm: + @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) build-arm || exit 1;) + # Requires login at https://docker.elastic.co:7000/. push: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) push || exit 1;) -.PHONY: build push +push-arm: + @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) push-arm || exit 1;) + +.PHONY: build build-arm push push-arm diff --git a/go/Makefile.debian12 b/go/Makefile.debian12 index 04ecdcdf..70e47008 100644 --- a/go/Makefile.debian12 +++ b/go/Makefile.debian12 @@ -1,4 +1,5 @@ -IMAGES := base main darwin arm armhf armel mips s390x darwin-arm64 npcap +IMAGES := base main darwin armhf armel mips s390x npcap +ARM_IMAGES := base-arm darwin-arm64 DEBIAN_VERSION := 12 TAG_EXTENSION := -debian12 @@ -7,8 +8,14 @@ export DEBIAN_VERSION TAG_EXTENSION build: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) build || exit 1;) +build-arm: + @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) build-arm || exit 1;) + # Requires login at https://docker.elastic.co:7000/. push: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) push || exit 1;) -.PHONY: build push +push-arm: + @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) push-arm || exit 1;) + +.PHONY: build build-arm push push-arm diff --git a/go/base-arm/Dockerfile.tmpl b/go/base-arm/Dockerfile.tmpl index 1dbff8e7..83aed790 100644 --- a/go/base-arm/Dockerfile.tmpl +++ b/go/base-arm/Dockerfile.tmpl @@ -25,13 +25,33 @@ RUN \ libxml2-dev \ libxml2 \ libicu-dev \ - libicu57 \ icu-devtools \ libsystemd-dev \ + {{- if eq .DEBIAN_VERSION "9" }} + libicu57 \ librpm3 \ librpmio3 \ librpmbuild3 \ librpmsign3 \ + {{- else if eq .DEBIAN_VERSION "10" }} + libicu63 \ + librpm8 \ + librpmio8 \ + librpmbuild8 \ + librpmsign8 \ + {{- else if eq .DEBIAN_VERSION "11" }} + libicu67 \ + librpm9 \ + librpmio9 \ + librpmbuild9 \ + librpmsign9 \ + {{- else }} + libicu72 \ + librpm9 \ + librpmio9 \ + librpmbuild9 \ + librpmsign9 \ + {{- end }} libsqlite3-dev \ libnss3 \ libsqlite3-0 \ @@ -77,7 +97,7 @@ RUN go mod init github.com/elastic/golang-crossbuild-$GOLANG_VERSION-arm \ && CGO_ENABLED=0 GOARCH=arm64 go build -o /crossbuild /entrypoint.go \ && rm -rf /go/* /root/.cache/* /entrypoint.go -COPY sdks/libpcap-1.8.1.tar.gz . +COPY sdks/libpcap-1.8.1.tar.gz . RUN mkdir /libpcap \ && tar -xzf libpcap-1.8.1.tar.gz -C /libpcap \ && cd /libpcap/libpcap-1.8.1 \ diff --git a/go/base-arm/sources-debian10.list b/go/base-arm/sources-debian10.list new file mode 100644 index 00000000..6b7ca118 --- /dev/null +++ b/go/base-arm/sources-debian10.list @@ -0,0 +1,3 @@ +# see https://wiki.debian.org/CrossToolchains +deb [arch=arm64] http://deb.debian.org/debian buster main +deb [arch=arm64] http://security.debian.org/debian-security buster/updates main diff --git a/go/base-arm/sources-debian11.list b/go/base-arm/sources-debian11.list new file mode 100644 index 00000000..54e8518c --- /dev/null +++ b/go/base-arm/sources-debian11.list @@ -0,0 +1,4 @@ +# see https://wiki.debian.org/CrossToolchains +deb [arch=arm64] http://deb.debian.org/debian bullseye main +deb [arch=arm64] http://deb.debian.org/debian bullseye-updates main +deb [arch=arm64] http://deb.debian.org/debian-security/ bullseye-security main diff --git a/go/base-arm/sources-debian12.list b/go/base-arm/sources-debian12.list new file mode 100644 index 00000000..95a0d636 --- /dev/null +++ b/go/base-arm/sources-debian12.list @@ -0,0 +1,3 @@ +# see https://wiki.debian.org/CrossToolchains +deb [arch=arm64] http://deb.debian.org/debian bookworm main +deb [arch=arm64] http://deb.debian.org/debian-security/ bookworm-security main diff --git a/go/base-arm/sources-debian9.list b/go/base-arm/sources-debian9.list index 812b5458..995788f6 100644 --- a/go/base-arm/sources-debian9.list +++ b/go/base-arm/sources-debian9.list @@ -1,2 +1,2 @@ -deb http://archive.debian.org/debian stretch main +deb [arch=arm64] http://archive.debian.org/debian stretch main deb [arch=arm64] http://archive.debian.org/debian-security stretch/updates main diff --git a/go/base/Dockerfile.tmpl b/go/base/Dockerfile.tmpl index db27eafd..68dfbeee 100644 --- a/go/base/Dockerfile.tmpl +++ b/go/base/Dockerfile.tmpl @@ -70,7 +70,7 @@ RUN go mod init github.com/elastic/golang-crossbuild-$GOLANG_VERSION \ && CGO_ENABLED=0 GOARCH=amd64 go build -o /crossbuild /entrypoint.go \ && rm -rf /go/* /root/.cache/* /entrypoint.go -COPY sdks/libpcap-1.8.1.tar.gz . +COPY sdks/libpcap-1.8.1.tar.gz . RUN mkdir /libpcap \ && tar -xzf libpcap-1.8.1.tar.gz -C /libpcap \ && rm libpcap-1.8.1.tar.gz diff --git a/go/darwin-arm64/Dockerfile.tmpl b/go/darwin-arm64/Dockerfile.tmpl index b7d359e4..f130b3a4 100644 --- a/go/darwin-arm64/Dockerfile.tmpl +++ b/go/darwin-arm64/Dockerfile.tmpl @@ -2,17 +2,9 @@ ARG REPOSITORY ARG VERSION ARG TAG_EXTENSION='' {{- if or (eq .DEBIAN_VERSION "10") (eq .DEBIAN_VERSION "11") (eq .DEBIAN_VERSION "12")}} -FROM --platform=linux/amd64 docker.elastic.co/beats-dev/golang-crossbuild:llvm-apple-debian{{ .DEBIAN_VERSION }}-amd64 AS build-llvm-apple-amd64 -FROM --platform=linux/arm64 docker.elastic.co/beats-dev/golang-crossbuild:llvm-apple-debian{{ .DEBIAN_VERSION }}-arm64 AS build-llvm-apple-arm64 -# workaround to https://github.com/moby/moby/issues/34482 -ARG TARGETARCH=amd64 -ARG BUILDARCH=amd64 -FROM build-llvm-apple-${TARGETARCH} as build-llvm-apple -ARG TARGETARCH -ARG BUILDARCH -RUN echo "Building ${TARGETARCH} on a ${BUILDARCH}" +FROM --platform=linux/arm64 docker.elastic.co/beats-dev/golang-crossbuild:llvm-apple-debian{{ .DEBIAN_VERSION }}-arm64 AS build-llvm-apple {{- end }} -FROM ${REPOSITORY}/golang-crossbuild:${VERSION}-base${TAG_EXTENSION} +FROM ${REPOSITORY}/golang-crossbuild:${VERSION}-base-arm${TAG_EXTENSION} {{- if and (ne .DEBIAN_VERSION "10") (ne .DEBIAN_VERSION "11") (ne .DEBIAN_VERSION "12")}} RUN echo "This Docker image will work only with Debian >10" && exit 1 diff --git a/go/darwin-arm64/rootfs/compilers.yaml b/go/darwin-arm64/rootfs/compilers.yaml index 297670b4..26f36667 100644 --- a/go/darwin-arm64/rootfs/compilers.yaml +++ b/go/darwin-arm64/rootfs/compilers.yaml @@ -1,9 +1,6 @@ --- darwin: - amd64: - CC: o64-clang - CXX: o64-clang++ arm64: CC: oa64-clang CXX: oa64-clang++ diff --git a/go/darwin/Dockerfile.tmpl b/go/darwin/Dockerfile.tmpl index 143579f5..d4e68674 100644 --- a/go/darwin/Dockerfile.tmpl +++ b/go/darwin/Dockerfile.tmpl @@ -3,15 +3,7 @@ ARG VERSION ARG TAG_EXTENSION='' {{- if or (eq .DEBIAN_VERSION "10") (eq .DEBIAN_VERSION "11") (eq .DEBIAN_VERSION "12")}} -FROM --platform=linux/amd64 docker.elastic.co/beats-dev/golang-crossbuild:llvm-apple-debian{{ .DEBIAN_VERSION }}-amd64 AS build-llvm-apple-amd64 -FROM --platform=linux/arm64 docker.elastic.co/beats-dev/golang-crossbuild:llvm-apple-debian{{ .DEBIAN_VERSION }}-arm64 AS build-llvm-apple-arm64 -# workaround to https://github.com/moby/moby/issues/34482 -ARG TARGETARCH=amd64 -ARG BUILDARCH=amd64 -FROM build-llvm-apple-${TARGETARCH} as build-llvm-apple -ARG TARGETARCH -ARG BUILDARCH -RUN echo "Building ${TARGETARCH} on a ${BUILDARCH}" +FROM --platform=linux/amd64 docker.elastic.co/beats-dev/golang-crossbuild:llvm-apple-debian{{ .DEBIAN_VERSION }}-amd64 AS build-llvm-apple {{- end }} FROM ${REPOSITORY}/golang-crossbuild:${VERSION}-base${TAG_EXTENSION}