diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 986859cf..ef110ee7 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -42,11 +42,11 @@ steps: - "false" - "true" - - label: ":linux: Staging / Ubuntu ARM - {{matrix.makefile}} - fips: {{matrix.fips}}" + - label: ":linux: Staging / Ubuntu ARM - Makefile.debian9 - fips: {{matrix.fips}}" key: "build-ubuntu-arm" command: - - ".buildkite/scripts/build.sh {{matrix.makefile}}" - - ".buildkite/scripts/publish.sh {{matrix.makefile}}" + - ".buildkite/scripts/build.sh Makefile.debian9" + - ".buildkite/scripts/publish.sh Makefile.debian9" env: REPOSITORY: "${STAGING_IMAGE}" FIPS: "{{matrix.fips}}" @@ -60,11 +60,6 @@ steps: instanceType: "t4g.large" matrix: setup: - makefile: - - "Makefile.debian9" - - "Makefile.debian10" - - "Makefile.debian11" - - "Makefile.debian12" fips: - "false" - "true" @@ -101,11 +96,11 @@ steps: - github_commit_status: context: "Release / Ubuntu X86_64" - - label: ":linux: Release / Ubuntu ARM - {{matrix.makefile}} - fips: {{matrix.fips}}" + - label: ":linux: Release / Ubuntu ARM - Makefile.debian9 - fips: {{matrix.fips}}" key: "release-ubuntu-arm" command: - - ".buildkite/scripts/build.sh {{matrix.makefile}}" - - ".buildkite/scripts/publish.sh {{matrix.makefile}}" + - ".buildkite/scripts/build.sh Makefile.debian9" + - ".buildkite/scripts/publish.sh Makefile.debian9" env: FIPS: "{{matrix.fips}}" # Releases should only be for main for ^[0-9].[0-9] branches (therefore support for major.minor.patch.x too). @@ -116,11 +111,6 @@ steps: instanceType: "t4g.large" matrix: setup: - makefile: - - "Makefile.debian9" - - "Makefile.debian10" - - "Makefile.debian11" - - "Makefile.debian12" fips: - "false" - "true" diff --git a/Makefile.common b/Makefile.common index de2b9e6d..5e0dd98b 100644 --- a/Makefile.common +++ b/Makefile.common @@ -9,6 +9,13 @@ SUFFIX_NPCAP_VERSION := -npcap-$(NPCAP_VERSION) NPCAP_REPOSITORY := docker.elastic.co/observability-ci GS_BUCKET_PATH ?= ingest-buildkite-ci + +ifeq ($(BUILDX),1) +ifeq ($(shell test $(DEBIAN_VERSION) -ge 10; echo $$?),0) +DOCKER_MULTIARCH := 1 +endif +endif + # Requires login at google storage. copy-npcap: ifeq ($(CI),true) diff --git a/go/Makefile.debian10 b/go/Makefile.debian10 index 82ae73b7..0ae5d8f4 100644 --- a/go/Makefile.debian10 +++ b/go/Makefile.debian10 @@ -1,5 +1,4 @@ -IMAGES := base main darwin armhf npcap -ARM_IMAGES := base-arm darwin-arm64 +IMAGES ?= base main darwin arm armhf darwin-arm64 npcap DEBIAN_VERSION := 10 TAG_EXTENSION := -debian10 @@ -8,14 +7,8 @@ export DEBIAN_VERSION TAG_EXTENSION build: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) build || exit 1;) -build-arm: - @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) build-arm || exit 1;) - # Requires login at https://docker.elastic.co:7000/. push: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) push || exit 1;) -push-arm: - @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) push-arm || exit 1;) - -.PHONY: build build-arm push push-arm +.PHONY: build push diff --git a/go/Makefile.debian11 b/go/Makefile.debian11 index f7631965..24da642c 100644 --- a/go/Makefile.debian11 +++ b/go/Makefile.debian11 @@ -1,5 +1,4 @@ -IMAGES := base main darwin armhf npcap -ARM_IMAGES := base-arm darwin-arm64 +IMAGES := base main darwin arm armhf darwin-arm64 npcap DEBIAN_VERSION := 11 TAG_EXTENSION := -debian11 @@ -8,14 +7,8 @@ export DEBIAN_VERSION TAG_EXTENSION build: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) build || exit 1;) -build-arm: - @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) build-arm || exit 1;) - # Requires login at https://docker.elastic.co:7000/. push: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) push || exit 1;) -push-arm: - @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) push-arm || exit 1;) - -.PHONY: build build-arm push push-arm +.PHONY: build push diff --git a/go/Makefile.debian12 b/go/Makefile.debian12 index 70e47008..04ecdcdf 100644 --- a/go/Makefile.debian12 +++ b/go/Makefile.debian12 @@ -1,5 +1,4 @@ -IMAGES := base main darwin armhf armel mips s390x npcap -ARM_IMAGES := base-arm darwin-arm64 +IMAGES := base main darwin arm armhf armel mips s390x darwin-arm64 npcap DEBIAN_VERSION := 12 TAG_EXTENSION := -debian12 @@ -8,14 +7,8 @@ export DEBIAN_VERSION TAG_EXTENSION build: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) build || exit 1;) -build-arm: - @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) build-arm || exit 1;) - # Requires login at https://docker.elastic.co:7000/. push: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) push || exit 1;) -push-arm: - @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) push-arm || exit 1;) - -.PHONY: build build-arm push push-arm +.PHONY: build push diff --git a/go/base-arm/Dockerfile.tmpl b/go/base-arm/Dockerfile.tmpl index be1b4a04..97375004 100644 --- a/go/base-arm/Dockerfile.tmpl +++ b/go/base-arm/Dockerfile.tmpl @@ -25,33 +25,13 @@ RUN \ libxml2-dev \ libxml2 \ libicu-dev \ + libicu57 \ icu-devtools \ libsystemd-dev \ - {{- if eq .DEBIAN_VERSION "9" }} - libicu57 \ librpm3 \ librpmio3 \ librpmbuild3 \ librpmsign3 \ - {{- else if eq .DEBIAN_VERSION "10" }} - libicu63 \ - librpm8 \ - librpmio8 \ - librpmbuild8 \ - librpmsign8 \ - {{- else if eq .DEBIAN_VERSION "11" }} - libicu67 \ - librpm9 \ - librpmio9 \ - librpmbuild9 \ - librpmsign9 \ - {{- else }} - libicu72 \ - librpm9 \ - librpmio9 \ - librpmbuild9 \ - librpmsign9 \ - {{- end }} libsqlite3-dev \ libnss3 \ libsqlite3-0 \ @@ -96,7 +76,7 @@ RUN go mod init github.com/elastic/golang-crossbuild-$GOLANG_VERSION-arm \ && go build -o /crossbuild /entrypoint.go \ && rm -rf /go/* /root/.cache/* /entrypoint.go -COPY sdks/libpcap-1.8.1.tar.gz . +COPY sdks/libpcap-1.8.1.tar.gz . RUN mkdir /libpcap \ && tar -xzf libpcap-1.8.1.tar.gz -C /libpcap \ && cd /libpcap/libpcap-1.8.1 \ diff --git a/go/base-arm/sources-debian10.list b/go/base-arm/sources-debian10.list deleted file mode 100644 index 6b7ca118..00000000 --- a/go/base-arm/sources-debian10.list +++ /dev/null @@ -1,3 +0,0 @@ -# see https://wiki.debian.org/CrossToolchains -deb [arch=arm64] http://deb.debian.org/debian buster main -deb [arch=arm64] http://security.debian.org/debian-security buster/updates main diff --git a/go/base-arm/sources-debian11.list b/go/base-arm/sources-debian11.list deleted file mode 100644 index 54e8518c..00000000 --- a/go/base-arm/sources-debian11.list +++ /dev/null @@ -1,4 +0,0 @@ -# see https://wiki.debian.org/CrossToolchains -deb [arch=arm64] http://deb.debian.org/debian bullseye main -deb [arch=arm64] http://deb.debian.org/debian bullseye-updates main -deb [arch=arm64] http://deb.debian.org/debian-security/ bullseye-security main diff --git a/go/base-arm/sources-debian12.list b/go/base-arm/sources-debian12.list deleted file mode 100644 index 95a0d636..00000000 --- a/go/base-arm/sources-debian12.list +++ /dev/null @@ -1,3 +0,0 @@ -# see https://wiki.debian.org/CrossToolchains -deb [arch=arm64] http://deb.debian.org/debian bookworm main -deb [arch=arm64] http://deb.debian.org/debian-security/ bookworm-security main diff --git a/go/base-arm/sources-debian9.list b/go/base-arm/sources-debian9.list index 995788f6..812b5458 100644 --- a/go/base-arm/sources-debian9.list +++ b/go/base-arm/sources-debian9.list @@ -1,2 +1,2 @@ -deb [arch=arm64] http://archive.debian.org/debian stretch main +deb http://archive.debian.org/debian stretch main deb [arch=arm64] http://archive.debian.org/debian-security stretch/updates main diff --git a/go/base/Dockerfile.tmpl b/go/base/Dockerfile.tmpl index 667c4b7e..bb4acab6 100644 --- a/go/base/Dockerfile.tmpl +++ b/go/base/Dockerfile.tmpl @@ -69,7 +69,7 @@ RUN go mod init github.com/elastic/golang-crossbuild-$GOLANG_VERSION \ && CGO_ENABLED=0 go build -o /crossbuild /entrypoint.go \ && rm -rf /go/* /root/.cache/* /entrypoint.go -COPY sdks/libpcap-1.8.1.tar.gz . +COPY sdks/libpcap-1.8.1.tar.gz . RUN mkdir /libpcap \ && tar -xzf libpcap-1.8.1.tar.gz -C /libpcap \ && rm libpcap-1.8.1.tar.gz diff --git a/go/darwin-arm64/Dockerfile.tmpl b/go/darwin-arm64/Dockerfile.tmpl index f130b3a4..b7d359e4 100644 --- a/go/darwin-arm64/Dockerfile.tmpl +++ b/go/darwin-arm64/Dockerfile.tmpl @@ -2,9 +2,17 @@ ARG REPOSITORY ARG VERSION ARG TAG_EXTENSION='' {{- if or (eq .DEBIAN_VERSION "10") (eq .DEBIAN_VERSION "11") (eq .DEBIAN_VERSION "12")}} -FROM --platform=linux/arm64 docker.elastic.co/beats-dev/golang-crossbuild:llvm-apple-debian{{ .DEBIAN_VERSION }}-arm64 AS build-llvm-apple +FROM --platform=linux/amd64 docker.elastic.co/beats-dev/golang-crossbuild:llvm-apple-debian{{ .DEBIAN_VERSION }}-amd64 AS build-llvm-apple-amd64 +FROM --platform=linux/arm64 docker.elastic.co/beats-dev/golang-crossbuild:llvm-apple-debian{{ .DEBIAN_VERSION }}-arm64 AS build-llvm-apple-arm64 +# workaround to https://github.com/moby/moby/issues/34482 +ARG TARGETARCH=amd64 +ARG BUILDARCH=amd64 +FROM build-llvm-apple-${TARGETARCH} as build-llvm-apple +ARG TARGETARCH +ARG BUILDARCH +RUN echo "Building ${TARGETARCH} on a ${BUILDARCH}" {{- end }} -FROM ${REPOSITORY}/golang-crossbuild:${VERSION}-base-arm${TAG_EXTENSION} +FROM ${REPOSITORY}/golang-crossbuild:${VERSION}-base${TAG_EXTENSION} {{- if and (ne .DEBIAN_VERSION "10") (ne .DEBIAN_VERSION "11") (ne .DEBIAN_VERSION "12")}} RUN echo "This Docker image will work only with Debian >10" && exit 1 diff --git a/go/darwin-arm64/rootfs/compilers.yaml b/go/darwin-arm64/rootfs/compilers.yaml index 26f36667..297670b4 100644 --- a/go/darwin-arm64/rootfs/compilers.yaml +++ b/go/darwin-arm64/rootfs/compilers.yaml @@ -1,6 +1,9 @@ --- darwin: + amd64: + CC: o64-clang + CXX: o64-clang++ arm64: CC: oa64-clang CXX: oa64-clang++ diff --git a/go/darwin/Dockerfile.tmpl b/go/darwin/Dockerfile.tmpl index d4e68674..143579f5 100644 --- a/go/darwin/Dockerfile.tmpl +++ b/go/darwin/Dockerfile.tmpl @@ -3,7 +3,15 @@ ARG VERSION ARG TAG_EXTENSION='' {{- if or (eq .DEBIAN_VERSION "10") (eq .DEBIAN_VERSION "11") (eq .DEBIAN_VERSION "12")}} -FROM --platform=linux/amd64 docker.elastic.co/beats-dev/golang-crossbuild:llvm-apple-debian{{ .DEBIAN_VERSION }}-amd64 AS build-llvm-apple +FROM --platform=linux/amd64 docker.elastic.co/beats-dev/golang-crossbuild:llvm-apple-debian{{ .DEBIAN_VERSION }}-amd64 AS build-llvm-apple-amd64 +FROM --platform=linux/arm64 docker.elastic.co/beats-dev/golang-crossbuild:llvm-apple-debian{{ .DEBIAN_VERSION }}-arm64 AS build-llvm-apple-arm64 +# workaround to https://github.com/moby/moby/issues/34482 +ARG TARGETARCH=amd64 +ARG BUILDARCH=amd64 +FROM build-llvm-apple-${TARGETARCH} as build-llvm-apple +ARG TARGETARCH +ARG BUILDARCH +RUN echo "Building ${TARGETARCH} on a ${BUILDARCH}" {{- end }} FROM ${REPOSITORY}/golang-crossbuild:${VERSION}-base${TAG_EXTENSION}