Cirrus: Reimplement release archive + upload

The initial implementation was far more complicated than necessary.
Strip out the complexities in favor of a simpler and more direct
approach.

Signed-off-by: Chris Evich <[email protected]>

Author: cevich

.cirrus.yml

@@ -62,6 +62,10 @@ env:
     GCE_SSH_USERNAME: cirrus-ci
     # Name where this repositories cloud resources are located
     GCP_PROJECT_ID: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f]
+    RELEASE_GCPJSON: ENCRYPTED[789d8f7e9a5972ce350fd8e60f1032ccbf4a35c3938b604774b711aad280e12c21faf10e25af1e0ba33597ffb9e39e46]
+    RELEASE_GCPNAME: ENCRYPTED[417d50488a4bd197bcc925ba6574de5823b97e68db1a17e3a5fde4bcf26576987345e75f8d9ea1c15a156b4612c072a1]
+    RELEASE_GCPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f]
+
 
 
 # Default VM to use unless set or modified by task
@@ -336,9 +340,8 @@ testing_task:
     unit_test_script: '$SCRIPT_BASE/unit_test.sh |& ${TIMESTAMP}'
     integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}'
     system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}'
-    cache_release_archive_script: >-
-        [[ "$TEST_REMOTE_CLIENT" == "false" ]] || \
-            $SCRIPT_BASE/cache_release_archive.sh |& ${TIMESTAMP}
+    build_release_script: '$SCRIPT_BASE/build_release.sh |& ${TIMESTAMP}'
+    upload_release_archive_script: '$SCRIPT_BASE/upload_release_archive.sh |& ${TIMESTAMP}'
 
     on_failure:
         failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh'
@@ -376,9 +379,6 @@ testing_crun_task:
     unit_test_script: '$SCRIPT_BASE/unit_test.sh |& ${TIMESTAMP}'
     integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}'
     system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}'
-    cache_release_archive_script: >-
-        [[ "$TEST_REMOTE_CLIENT" == "false" ]] || \
-            $SCRIPT_BASE/cache_release_archive.sh |& ${TIMESTAMP}
 
     on_failure:
         failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh'
@@ -459,14 +459,15 @@ special_testing_cross_task:
 
     env:
         matrix:
-            SPECIALMODE: 'windows'  # See docs
-            SPECIALMODE: 'darwin'
+            CROSS_PLATFORM: 'windows'
+            CROSS_PLATFORM: 'darwin'
 
     timeout_in: 20m
 
     networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh'
     setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
-    cache_release_archive_script: '$SCRIPT_BASE/cache_release_archive.sh |& ${TIMESTAMP}'
+    build_release_script: '$SCRIPT_BASE/build_release.sh |& ${TIMESTAMP}'
+    upload_release_archive_script: '$SCRIPT_BASE/upload_release_archive.sh |& ${TIMESTAMP}'
 
     on_failure:
         failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh'
@@ -615,6 +616,9 @@ verify_test_built_images_task:
     integration_test_script: >-
         [[ "$PACKER_BUILDER_NAME" == "xfedora-30" ]] || \
             $SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}
+    build_release_script: >-
+        [[ "$PACKER_BUILDER_NAME" == "xfedora-30" ]] || \
+            '$SCRIPT_BASE/build_release.sh |& ${TIMESTAMP}'
     system_test_script: >-
         [[ "$PACKER_BUILDER_NAME" == "xfedora-30" ]] || \
             $SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}
@@ -629,7 +633,7 @@ success_task:
     # it blocks PRs from merging if a depends_on task fails
     only_if: $CIRRUS_BRANCH != $DEST_BRANCH
 
-    # ignores any dependent task conditions, include everything except 'release'
+    # ignores any dependent task conditions
     depends_on:
         - "gating"
         - "vendor"
@@ -660,49 +664,3 @@ success_task:
         memory: 1
 
     success_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/success.sh |& ${TIMESTAMP}'
-
-
-release_task:
-
-    # Never do this when building images
-    only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*'
-
-    # TODO: Uncomment both to not affect pass/fail status of entire job?
-    # allow_failures: $CI == "true"
-    # skip_notifications: $CI == "true"
-
-    # Must include everything (YAML anchor/alias cannot be used here)
-    depends_on:
-        - "gating"
-        - "vendor"
-        - "varlink_api"
-        - "build_each_commit"
-        - "build_without_cgo"
-        - "meta"
-        - "image_prune"
-        - "testing"
-        - "testing_crun"
-        - "special_testing_rootless"
-        - "special_testing_in_podman"
-        - "special_testing_cgroupv2"
-        - "special_testing_cross"
-        - "special_testing_endpoint"
-        - "test_build_cache_images"
-        - "test_building_snap"
-        - "verify_test_built_images"
-        - "success"
-
-    gce_instance:
-        image_name: "${IMAGE_BUILDER_CACHE_IMAGE_NAME}"
-
-    timeout_in: 30m
-
-    env:
-        GCPJSON: ENCRYPTED[789d8f7e9a5972ce350fd8e60f1032ccbf4a35c3938b604774b711aad280e12c21faf10e25af1e0ba33597ffb9e39e46]
-        GCPNAME: ENCRYPTED[417d50488a4bd197bcc925ba6574de5823b97e68db1a17e3a5fde4bcf26576987345e75f8d9ea1c15a156b4612c072a1]
-        GCPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f]
-
-    uncache_release_archives_script: '$SCRIPT_BASE/uncache_release_archives.sh |& ${TIMESTAMP}'
-
-    on_failure:
-        failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh'

.gitignore

@@ -20,6 +20,7 @@ __pycache__
 /cmd/podman/varlink/iopodman.go
 .gopathok
 test/e2e/e2e.coverprofile
-/podman*zip
+release.txt
+podman-remote*.zip
 podman*.tar.gz
 .idea*

Makefile

@@ -81,13 +81,12 @@ LIBSECCOMP_COMMIT := release-2.3
 # caller may override in special circumstances if needed.
 GINKGOTIMEOUT ?= -timeout=90m
 
-RELEASE_VERSION ?= $(shell git fetch --tags && git describe HEAD 2> /dev/null)
-RELEASE_NUMBER ?= $(shell echo $(RELEASE_VERSION) | sed 's/-.*//')
-RELEASE_DIST ?= $(shell ( source /etc/os-release; echo $$ID ))
-RELEASE_DIST_VER ?= $(shell ( source /etc/os-release; echo $$VERSION_ID | cut -d '.' -f 1))
-RELEASE_ARCH ?= $(shell go env GOARCH 2> /dev/null)
-RELEASE_BASENAME := $(shell basename $(PROJECT))
-
+RELEASE_VERSION ?= $(shell hack/get_release_info.sh VERSION)
+RELEASE_NUMBER ?= $(shell hack/get_release_info.sh NUMBER)
+RELEASE_DIST ?= $(shell hack/get_release_info.sh DIST)
+RELEASE_DIST_VER ?= $(shell hack/get_release_info.sh DIST_VER)
+RELEASE_ARCH ?= $(shell hack/get_release_info.sh ARCH)
+RELEASE_BASENAME := $(shell hack/get_release_info.sh BASENAME)
 
 # If GOPATH not specified, use one in the local directory
 ifeq ($(GOPATH),)
@@ -164,11 +163,9 @@ podman: .gopathok $(PODMAN_VARLINK_DEPENDENCIES) ## Build with podman
 podman-remote: .gopathok $(PODMAN_VARLINK_DEPENDENCIES) ## Build with podman on remote environment
 	$(GO_BUILD) $(BUILDFLAGS) -gcflags '$(GCFLAGS)' -asmflags '$(ASMFLAGS)' -ldflags '$(LDFLAGS_PODMAN)' -tags "$(BUILDTAGS) remoteclient" -o bin/$@ $(PROJECT)/cmd/podman
 
-podman-remote-darwin: .gopathok $(PODMAN_VARLINK_DEPENDENCIES) ## Build with podman on remote OSX environment
-	CGO_ENABLED=0 GOOS=darwin $(GO_BUILD) -gcflags '$(GCFLAGS)' -asmflags '$(ASMFLAGS)' -ldflags '$(LDFLAGS_PODMAN)' -tags "remoteclient containers_image_openpgp exclude_graphdriver_devicemapper" -o bin/$@ $(PROJECT)/cmd/podman
-
-podman-remote-windows: .gopathok $(PODMAN_VARLINK_DEPENDENCIES) ## Build with podman for a remote windows environment
-	CGO_ENABLED=0 GOOS=windows $(GO_BUILD) -gcflags '$(GCFLAGS)' -asmflags '$(ASMFLAGS)' -ldflags '$(LDFLAGS_PODMAN)' -tags "remoteclient containers_image_openpgp exclude_graphdriver_devicemapper" -o bin/$@.exe $(PROJECT)/cmd/podman
+podman-remote-%: .gopathok $(PODMAN_VARLINK_DEPENDENCIES) ## Build podman for a specific GOOS
+	$(eval BINSFX := $(shell test "$*" != "windows" || echo ".exe"))
+	CGO_ENABLED=0 GOOS=$* $(GO_BUILD) -gcflags '$(GCFLAGS)' -asmflags '$(ASMFLAGS)' -ldflags '$(LDFLAGS_PODMAN)' -tags "remoteclient containers_image_openpgp exclude_graphdriver_devicemapper" -o bin/$@$(BINSFX) $(PROJECT)/cmd/podman
 
 local-cross: $(CROSS_BUILD_TARGETS) ## Cross local compilation
 
@@ -182,8 +179,9 @@ clean: ## Clean artifacts
 	rm -rf \
 		.gopathok \
 		_output \
-		podman*.zip \
-		podman*.tar.gz \
+		release.txt
+		$(wildcard podman-remote*.zip) \
+		$(wildcard podman*.tar.gz) \
 		bin \
 		build \
 		docs/remote \
@@ -300,23 +298,6 @@ vagrant-check:
 
 binaries: varlink_generate podman podman-remote  ## Build podman
 
-# Zip archives are supported on all platforms + allows embedding metadata
-podman.zip: binaries docs
-	$(eval TMPDIR := $(shell mktemp -d -p '' $@_XXXX))
-	test -n "$(TMPDIR)"
-	$(MAKE) install "DESTDIR=$(TMPDIR)" "PREFIX=$(TMPDIR)/usr"
-	# Encoded RELEASE_INFO format depended upon by CI tooling
-	# X-RELEASE-INFO format depended upon by CI tooling
-	cd "$(TMPDIR)" && echo \
-		"X-RELEASE-INFO: $(RELEASE_BASENAME) $(RELEASE_VERSION) $(RELEASE_DIST) $(RELEASE_DIST_VER) $(RELEASE_ARCH)" | \
-		zip --recurse-paths --archive-comment "$(CURDIR)/$@" "./"
-	-rm -rf "$(TMPDIR)"
-
-podman-remote-%.zip: podman-remote-%
-	# Don't label darwin/windows cros-compiles with local distribution & version
-	echo "X-RELEASE-INFO: podman-remote $(RELEASE_VERSION) $* cc $(RELEASE_ARCH)" | \
-        zip --archive-comment "$(CURDIR)/$@" ./bin/$<*
-
 install.catatonit:
 	./hack/install_catatonit.sh
 
@@ -333,19 +314,58 @@ docs: $(MANPAGES) ## Generate documentation
 install-podman-remote-docs: docs
 	@(cd docs; ./podman-remote.sh ./remote)
 
+# When publishing releases include critical build-time details
+.PHONY: release.txt
+release.txt:
+	# X-RELEASE-INFO format depended upon by automated tooling
+	echo -n "X-RELEASE-INFO:" > "$@"
+	for field in "$(RELEASE_BASENAME)" "$(RELEASE_VERSION)" \
+		         "$(RELEASE_DIST)" "$(RELEASE_DIST_VER)" "$(RELEASE_ARCH)"; do \
+		echo -n " $$field"; done >> "$@"
+	echo "" >> "$@"
+
+podman-$(RELEASE_NUMBER).tar.gz: binaries docs release.txt
+	$(eval TMPDIR := $(shell mktemp -d -p '' podman_XXXX))
+	$(eval SUBDIR := podman-$(RELEASE_NUMBER))
+	mkdir -p "$(TMPDIR)/$(SUBDIR)"
+	$(MAKE) install.bin install.man install.cni install.systemd "DESTDIR=$(TMPDIR)/$(SUBDIR)" "PREFIX=/usr"
+	# release.txt location and content depended upon by automated tooling
+	cp release.txt "$(TMPDIR)/"
+	tar -czvf $@ --xattrs -C "$(TMPDIR)" "./release.txt" "./$(SUBDIR)"
+	-rm -rf "$(TMPDIR)"
+
+# Must call make in-line: Dependency-spec. w/ wild-card also consumes variable value.
+podman-remote-$(RELEASE_NUMBER)-%.zip:
+	$(MAKE) podman-remote-$* install-podman-remote-docs release.txt \
+		RELEASE_BASENAME=$(shell hack/get_release_info.sh REMOTENAME) \
+		RELEASE_DIST=$* RELEASE_DIST_VER="-"
+	$(eval TMPDIR := $(shell mktemp -d -p '' $podman_remote_XXXX))
+	$(eval SUBDIR := podman-$(RELEASE_VERSION))
+	$(eval BINSFX := $(shell test "$*" != "windows" || echo ".exe"))
+	mkdir -p "$(TMPDIR)/$(SUBDIR)"
+	# release.txt location and content depended upon by automated tooling
+	cp release.txt "$(TMPDIR)/"
+	cp ./bin/podman-remote-$*$(BINSFX) "$(TMPDIR)/$(SUBDIR)/podman$(BINSFX)"
+	cp -r ./docs/remote "$(TMPDIR)/$(SUBDIR)/docs/"
+	$(eval DOCFILE := $(TMPDIR)/$(SUBDIR)/docs/podman.1)
+	cp docs/podman-remote.1 "$(DOCFILE)"
+	sed -i 's/podman\\*-remote/podman/g' "$(DOCFILE)"
+	sed -i 's/Podman\\*-remote/Podman\ for\ $*/g' "$(DOCFILE)"
+	sed -i 's/podman\.conf/podman\-remote\.conf/g' "$(DOCFILE)"
+	sed -i 's/A\ remote\ CLI\ for\ Podman\:\ //g' "$(DOCFILE)"
+	cd "$(TMPDIR)" && \
+		zip --recurse-paths "$(CURDIR)/$@" "./release.txt" "./"
+	-rm -rf "$(TMPDIR)"
+
+.PHONY: podman-release
+podman-release:
+	rm -f release.txt
+	$(MAKE) podman-$(RELEASE_NUMBER).tar.gz
 
-brew-pkg: install-podman-remote-docs podman-remote-darwin
-	@mkdir -p ./brew
-	@cp ./bin/podman-remote-darwin ./brew/podman
-	@cp -r ./docs/remote ./brew/docs/
-	@cp docs/podman-remote.1 ./brew/docs/podman.1
-	@cp docs/podman-remote.conf.5 ./brew/docs/podman-remote.conf.5
-	@sed -i 's/podman\\*-remote/podman/g' ./brew/docs/podman.1
-	@sed -i 's/Podman\\*-remote/Podman\ for\ Mac/g' ./brew/docs/podman.1
-	@sed -i 's/podman\.conf/podman\-remote\.conf/g' ./brew/docs/podman.1
-	@sed -i 's/A\ remote\ CLI\ for\ Podman\:\ //g' ./brew/docs/podman.1
-	tar -czvf podman-${RELEASE_NUMBER}.tar.gz ./brew
-	@rm -rf ./brew
+.PHONY: podman-remote-%-release
+podman-remote-%-release:
+	rm -f release.txt
+	$(MAKE) podman-remote-$(RELEASE_NUMBER)-$*.zip
 
 docker-docs: docs
 	(cd docs; ./dckrman.sh *.1)

README.md

@@ -8,14 +8,9 @@ popularized by Kubernetes.  Libpod also contains the Pod Manager tool `(Podman)`
 * [Latest Version: 1.4.4](https://github.com/containers/libpod/releases/latest)
 * [Continuous Integration:](contrib/cirrus/README.md) [![Build Status](https://api.cirrus-ci.com/github/containers/libpod.svg)](https://cirrus-ci.com/github/containers/libpod/master)
 * [GoDoc: ![GoDoc](https://godoc.org/github.com/containers/libpod/libpod?status.svg)](https://godoc.org/github.com/containers/libpod/libpod)
-* Latest, automatic binary release downloads:
-  * Master Branch:
-    * [Fedora 30](https://storage.cloud.google.com/libpod-master-releases/libpod-master-fedora-30-amd64.zip)
-    * [Fedora 29](https://storage.cloud.google.com/libpod-master-releases/libpod-master-fedora-29-amd64.zip)
-    * [Ubuntu 18](https://storage.cloud.google.com/libpod-master-releases/libpod-master-ubuntu-18-amd64.zip)
-    * [Ubuntu 19](https://storage.cloud.google.com/libpod-master-releases/libpod-master-ubuntu-19-amd64.zip)
-    * [Windows Client](https://storage.cloud.google.com/libpod-master-releases/podman-remote-master-windows-cc-amd64.zip)
-    * [MacOS Client](https://storage.cloud.google.com/libpod-master-releases/podman-remote-master-darwin-cc-amd64.zip)
+* Automated continuous release downloads (including remote-client):
+  * Master Branch: [https://storage.cloud.google.com/libpod-master-releases/](https://storage.cloud.google.com/libpod-master-releases/)
+  * Pull-requests: [https://storage.cloud.google.com/libpod-pr-releases/](https://storage.cloud.google.com/libpod-pr-releases/)
 
 ## Overview and scope
 

contrib/cirrus/build_release.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+source $(dirname $0)/lib.sh
+
+req_env_var TEST_REMOTE_CLIENT OS_RELEASE_ID GOSRC
+
+cd $GOSRC
+
+if [[ "$TEST_REMOTE_CLIENT" == "true" ]] && [[ -z "$CROSS_PLATFORM" ]]
+then
+    CROSS_PLATFORM=linux
+fi
+
+if [[ -n "$CROSS_PLATFORM" ]]
+then
+    echo "Compiling podman-remote release archive for ${CROSS_PLATFORM}"
+    case "$CROSS_PLATFORM" in
+        linux) ;&
+        windows) ;&
+        darwin)
+            make podman-remote-${CROSS_PLATFORM}-release
+            ;;
+        *)
+            die 1 "Unknown/unsupported cross-compile platform '$CROSS_PLATFORM'"
+            ;;
+    esac
+else
+    echo "Compiling release archive for $OS_RELEASE_ID"
+    make podman-release
+fi