Improve container build performance w/ caching

.cirrus.yml

@@ -98,12 +98,13 @@ container_images_task: &container_images
             TARGET_NAME: 'skopeo_cidev'
             DEST_FQIN: *fqin
     env: &image_env
-        TEMPDIR: '$CIRRUS_WORKING_DIR'
         # For quay.io/libpod namespace
         REG_USERNAME: ENCRYPTED[de755aef351c501ee480231c24eae25b15e2b2a2b7c629f477c1d427fc5269e360bb358a53bd8914605bae588e99b52a]
         REG_PASSWORD: ENCRYPTED[52268944bb0d6642c33efb1c5d7fb82d0c40f9e6988448de35827f9be2cc547c1383db13e8b21516dbd7a0a69a7ae536]
     script: ci/make_container_images.sh;
-
+    package_cache: &package_cache
+        folder: "/tmp/automation_images_tmp/.cache/**"
+        fingerprint_key: "${TARGET_NAME}-cache-version-1"
 
 tooling_images_task:
     alias: tooling_images
@@ -142,6 +143,7 @@ tooling_images_task:
         <<: *image_env
         PUSH_LATEST: 1  # scripts force to 0 if $CIRRUS_PR
     script: ci/make_container_images.sh;
+    package_cache: *package_cache
 
 
 base_images_task:

.gitignore

@@ -1 +1,2 @@
 */*.json
+/.cache

Makefile

@@ -90,26 +90,30 @@ ci_debug: $(_TEMPDIR)/ci_debug.tar ## Build and enter container for local develo
 		-e TEMPDIR=$(_TEMPDIR) \
 		docker-archive:$<
 
+# Takes 4 arguments: export filepath, FQIN, context dir, package cache key
 define podman_build
 	podman build -t $(2) \
-		-v $(_TEMPDIR)/var_cache_dnf:/var/cache/dnf:Z \
+		-v $(_TEMPDIR)/.cache/$(4):/var/cache/dnf:Z \
+		-v $(_TEMPDIR)/.cache/$(4):/var/cache/apt:Z \
 		--build-arg PACKER_VERSION=$(call err_if_empty,PACKER_VERSION) \
 		-f $(3)/Containerfile .
 	rm -f $(1)
 	podman save --quiet -o $(1) $(2)
-	podman rmi $(2)
 endef
 
-$(_TEMPDIR)/ci_debug.tar: $(_TEMPDIR)/var_cache_dnf ci/Containerfile ci/install_packages.txt ci/install_packages.sh lib.sh
-	$(call podman_build,$@,ci_debug,ci)
+$(_TEMPDIR)/ci_debug.tar: $(_TEMPDIR)/.cache/fedora ci/Containerfile ci/install_packages.txt ci/install_packages.sh lib.sh
+	$(call podman_build,$@,ci_debug,ci,fedora)
 
 $(_TEMPDIR):
 	mkdir -p $@
 
 $(_TEMPDIR)/bin: $(_TEMPDIR)
 	mkdir -p $@
 
-$(_TEMPDIR)/var_cache_dnf: $(_TEMPDIR)
+$(_TEMPDIR)/.cache: $(_TEMPDIR)
+	mkdir -p $@
+
+$(_TEMPDIR)/.cache/%: $(_TEMPDIR)/.cache
 	mkdir -p $@
 
 $(_TEMPDIR)/packer.zip: $(_TEMPDIR)
@@ -181,8 +185,8 @@ image_builder_debug: $(_TEMPDIR)/image_builder_debug.tar ## Build and enter cont
 		-e GAC_FILEPATH=$(call err_if_empty,GAC_FILEPATH) \
 		docker-archive:$<
 
-$(_TEMPDIR)/image_builder_debug.tar: $(_TEMPDIR) $(_TEMPDIR)/var_cache_dnf image_builder/Containerfile image_builder/install_packages.txt ci/install_packages.sh lib.sh
-	$(call podman_build,$@,image_builder_debug,image_builder)
+$(_TEMPDIR)/image_builder_debug.tar: $(_TEMPDIR)/.cache/centos image_builder/Containerfile image_builder/install_packages.txt ci/install_packages.sh lib.sh
+	$(call podman_build,$@,image_builder_debug,image_builder,centos)
 
 .PHONY: base_images
 # This needs to run in a virt/nested-virt capable environment
@@ -205,75 +209,67 @@ endef
 
 .PHONY: fedora_podman
 fedora_podman:  ## Build Fedora podman development container
-	$(call build_podman_container,$@)
+	$(call build_podman_container,$@,fedora)
 
 .PHONY: prior-fedora_podman
 prior-fedora_podman:  ## Build Prior-Fedora podman development container
-	$(call build_podman_container,$@)
+	$(call build_podman_container,$@,prior-fedora)
 
 .PHONY: ubuntu_podman
 ubuntu_podman:  ## Build Ubuntu podman development container
-	$(call build_podman_container,$@)
+	$(call build_podman_container,$@,ubuntu)
 
-$(_TEMPDIR)/%_podman.tar: podman/Containerfile podman/setup.sh $(wildcard base_images/*.sh) $(wildcard cache_images/*.sh) $(_TEMPDIR) $(_TEMPDIR)/var_cache_dnf
+$(_TEMPDIR)/%_podman.tar: podman/Containerfile podman/setup.sh $(wildcard base_images/*.sh) $(wildcard cache_images/*.sh) $(_TEMPDIR)/.cache/%
 	podman build -t $*_podman:$(call err_if_empty,IMG_SFX) \
 		--build-arg=BASE_NAME=$(subst prior-,,$*) \
 		--build-arg=BASE_TAG=$(call err_if_empty,BASE_TAG) \
 		--build-arg=PACKER_BUILD_NAME=$(subst _podman,,$*) \
-		-v $(_TEMPDIR)/var_cache_dnf:/var/cache/dnf:Z \
-		-v $(_TEMPDIR)/var_cache_dnf:/var/cache/apt:Z \
+		-v $(_TEMPDIR)/.cache/$*:/var/cache/dnf:Z \
+		-v $(_TEMPDIR)/.cache/$*:/var/cache/apt:Z \
 		-f podman/Containerfile .
 	rm -f $@
 	podman save --quiet -o $@ $*_podman:$(IMG_SFX)
 
 .PHONY: skopeo_cidev
 skopeo_cidev: $(_TEMPDIR)/skopeo_cidev.tar  ## Build Skopeo development and CI container
-
-$(_TEMPDIR)/skopeo_cidev.tar: podman/fedora_release $(wildcard skopeo_base/*) $(_TEMPDIR) $(_TEMPDIR)/var_cache_dnf
+$(_TEMPDIR)/skopeo_cidev.tar: podman/fedora_release $(wildcard skopeo_base/*) $(_TEMPDIR)/.cache/fedora
 	podman build -t skopeo_cidev:$(call err_if_empty,IMG_SFX) \
 		--build-arg=BASE_TAG=$(_fedora_podman_release) \
-		-v $(_TEMPDIR)/var_cache_dnf:/var/cache/dnf:Z \
+		-v $(_TEMPDIR)/.cache/fedora:/var/cache/dnf:Z \
 		skopeo_cidev
 	rm -f $@
 	podman save --quiet -o $@ skopeo_cidev:$(IMG_SFX)
 
 .PHONY: imgts
 imgts: $(_TEMPDIR)/imgts.tar  ## Build the VM image time-stamping container image
-$(_TEMPDIR)/imgts.tar: imgts/Containerfile imgts/entrypoint.sh imgts/google-cloud-sdk.repo imgts/lib_entrypoint.sh $(_TEMPDIR)
-	podman build -t imgts:$(call err_if_empty,IMG_SFX) \
-		-f imgts/Containerfile .
-	rm -f $@
-	podman save --quiet -o $@ imgts:$(IMG_SFX)
+$(_TEMPDIR)/imgts.tar: imgts/Containerfile imgts/entrypoint.sh imgts/google-cloud-sdk.repo imgts/lib_entrypoint.sh $(_TEMPDIR)/.cache/centos
+	$(call podman_build,$@,imgts:$(call err_if_empty,IMG_SFX),imgts,centos)
 
-.PHONY: imgobsolete
-imgobsolete: $(_TEMPDIR)/imgobsolete.tar  ## Build the VM Image obsoleting container image
-$(_TEMPDIR)/imgobsolete.tar: $(_TEMPDIR)/imgts.tar imgts/lib_entrypoint.sh imgobsolete/Containerfile imgobsolete/entrypoint.sh $(_TEMPDIR)
+define imgts_base_podman_build
 	podman load -i $(_TEMPDIR)/imgts.tar
 	podman tag imgts:$(call err_if_empty,IMG_SFX) imgts:latest
-	podman build -t imgobsolete:$(call err_if_empty,IMG_SFX) \
-		-f imgobsolete/Containerfile .
-	rm -f $@
-	podman save --quiet -o $@ imgobsolete:$(IMG_SFX)
+	$(call podman_build,$@,$(1):$(call err_if_empty,IMG_SFX),$(1),centos)
+endef
+
+.PHONY: imgobsolete
+imgobsolete: $(_TEMPDIR)/imgobsolete.tar  ## Build the VM Image obsoleting container image
+$(_TEMPDIR)/imgobsolete.tar: $(_TEMPDIR)/imgts.tar imgts/lib_entrypoint.sh imgobsolete/Containerfile imgobsolete/entrypoint.sh $(_TEMPDIR)/.cache/centos
+	$(call imgts_base_podman_build,imgobsolete)
 
 .PHONY: imgprune
 imgprune: $(_TEMPDIR)/imgprune.tar  ## Build the VM Image pruning container image
-$(_TEMPDIR)/imgprune.tar: $(_TEMPDIR)/imgts.tar imgts/lib_entrypoint.sh imgprune/Containerfile imgprune/entrypoint.sh $(_TEMPDIR)
-	podman load -i $(_TEMPDIR)/imgts.tar
-	podman tag imgts:$(call err_if_empty,IMG_SFX) imgts:latest
-	podman build -t imgprune:$(call err_if_empty,IMG_SFX) \
-		-f imgprune/Containerfile .
-	rm -f $@
-	podman save --quiet -o $@ imgprune:$(IMG_SFX)
+$(_TEMPDIR)/imgprune.tar: $(_TEMPDIR)/imgts.tar imgts/lib_entrypoint.sh imgprune/Containerfile imgprune/entrypoint.sh $(_TEMPDIR)/.cache/centos
+	$(call imgts_base_podman_build,imgprune)
 
 .PHONY: gcsupld
 gcsupld: $(_TEMPDIR)/gcsupld.tar  ## Build the GCS Upload container image
-$(_TEMPDIR)/gcsupld.tar: $(_TEMPDIR)/imgts.tar imgts/lib_entrypoint.sh gcsupld/Containerfile gcsupld/entrypoint.sh $(_TEMPDIR)
-	podman load -i $(_TEMPDIR)/imgts.tar
-	podman tag imgts:$(call err_if_empty,IMG_SFX) imgts:latest
-	podman build -t gcsupld:$(call err_if_empty,IMG_SFX) \
-		-f gcsupld/Containerfile .
-	rm -f $@
-	podman save --quiet -o $@ gcsupld:$(IMG_SFX)
+$(_TEMPDIR)/gcsupld.tar: $(_TEMPDIR)/imgts.tar imgts/lib_entrypoint.sh gcsupld/Containerfile gcsupld/entrypoint.sh $(_TEMPDIR)/.cache/centos
+	$(call imgts_base_podman_build,gcsupld)
+
+.PHONY: orphanvms
+orphanvms: $(_TEMPDIR)/orphanvms.tar  ## Build the Orphaned VM container image
+$(_TEMPDIR)/orphanvms.tar: $(_TEMPDIR)/imgts.tar imgts/lib_entrypoint.sh orphanvms/Containerfile orphanvms/entrypoint.sh $(_TEMPDIR)/.cache/centos
+	$(call imgts_base_podman_build,orphanvms)
 
 .PHONY: .get_ci_vm
 get_ci_vm: $(_TEMPDIR)/get_ci_vm.tar  ## Build the get_ci_vm container image
@@ -282,20 +278,9 @@ $(_TEMPDIR)/get_ci_vm.tar: lib.sh get_ci_vm/Containerfile get_ci_vm/entrypoint.s
 	rm -f $@
 	podman save --quiet -o $@ get_ci_vm:$(IMG_SFX)
 
-.PHONY: orphanvms
-orphanvms: $(_TEMPDIR)/orphanvms.tar  ## Build the Orphaned VM container image
-$(_TEMPDIR)/orphanvms.tar: $(_TEMPDIR)/imgts.tar imgts/lib_entrypoint.sh orphanvms/Containerfile orphanvms/entrypoint.sh $(_TEMPDIR)
-	podman load -i $(_TEMPDIR)/imgts.tar
-	podman tag imgts:$(call err_if_empty,IMG_SFX) imgts:latest
-	podman build -t orphanvms:$(IMG_SFX) \
-		-f orphanvms/Containerfile .
-	rm -f $@
-	podman save --quiet -o $@ orphanvms:$(IMG_SFX)
-
-
 .PHONY: clean
 clean: ## Remove all generated files referenced in this Makefile
-	-rm -vrf $(_TEMPDIR)
+	-rm -rf $(_TEMPDIR)
 	-rm -f image_builder/*.json
 	-rm -f base_images/{*.json,cidata*,*-data}
 	-rm -f ci_debug.tar

ci/Containerfile

@@ -27,6 +27,7 @@ RUN dnf update -y && \
         gawk \
         jq \
         make \
+        openssh-clients \
         python3-pyyaml \
         rootfiles \
         rsync \

gcsupld/Containerfile

@@ -1,7 +1,6 @@
 FROM imgts:latest
 
-RUN yum -y update && \
-    yum clean all
+RUN yum -y --setopt=keepcache=true update
 
 COPY /gcsupld/entrypoint.sh /usr/local/bin/entrypoint.sh
 RUN chmod 755 /usr/local/bin/entrypoint.sh

imgobsolete/Containerfile

@@ -1,7 +1,6 @@
 FROM imgts:latest
 
-RUN yum -y update && \
-    yum clean all
+RUN yum -y --setopt=keepcache=true update
 
 COPY /imgobsolete/entrypoint.sh /usr/local/bin/entrypoint.sh
 RUN chmod 755 /usr/local/bin/entrypoint.sh

imgprune/Containerfile

@@ -1,7 +1,6 @@
 FROM imgts:latest
 
-RUN yum -y update && \
-    yum clean all
+RUN yum -y --setopt=keepcache=true update
 
 COPY /imgprune/entrypoint.sh /usr/local/bin/entrypoint.sh
 RUN chmod 755 /usr/local/bin/entrypoint.sh

imgts/Containerfile

@@ -2,10 +2,9 @@ FROM quay.io/centos/centos:stream8
 
 # Only needed for installing build-time dependencies
 COPY /imgts/google-cloud-sdk.repo /etc/yum.repos.d/google-cloud-sdk.repo
-RUN yum -y update && \
-    yum -y install epel-release && \
-    yum -y install google-cloud-sdk && \
-    yum clean all
+RUN yum -y --setopt=keepcache=true update && \
+    yum -y --setopt=keepcache=true install epel-release && \
+    yum -y --setopt=keepcache=true install google-cloud-sdk
 
 # These all represent required variables which must be set by caller
 ENV GCPJSON="__unknown__" \

lib.sh

@@ -174,8 +174,7 @@ common_finalize() {
 rh_finalize() {
     set +e  # Don't fail at the very end
     set -x
-    $SUDO dnf clean all
-    $SUDO rm -rf /var/cache/{yum,dnf}
+    # Packaging cache is preserved across builds of container images
     $SUDO rm -f /etc/udev/rules.d/*-persistent-*.rules
     $SUDO touch /.unconfigured  # force firstboot to run
     common_finalize
@@ -185,9 +184,7 @@ rh_finalize() {
 ubuntu_finalize() {
     set +e  # Don't fail at the very end
     set -x
-    $SUDO apt-get -qq -y autoremove
-    $SUDO rm -f /var/cache/apt/*  # DON'T remove archives: packages downloaded there
-    $SUDO rm -rf /var/lib/apt/lists/*
+    # Packaging cache is preserved across builds of container images
     common_finalize
 }
 

orphanvms/Containerfile

@@ -1,7 +1,6 @@
 FROM imgts:latest
 
-RUN yum -y update && \
-    yum clean all
+RUN yum -y --setopt=keepcache=true update
 
 COPY /orphanvms/entrypoint.sh /usr/local/bin/entrypoint.sh
 RUN chmod 755 /usr/local/bin/entrypoint.sh