From 0def6ba9999c7a30f136808bb0f7c965ce7f7b8e Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Wed, 21 Jul 2021 13:12:11 -0400 Subject: [PATCH] Improve container build performance w/ caching A substantial amount of time is spent refreshing packaging metadata and downloading packages during build. Since consecutive builds from essentially the same contents does this repeatedly, duplicating all that work is redundant. This set of changes causes Cirrus-CI to maintain a cache of packages and metadata for re-use at container build time. Signed-off-by: Chris Evich --- .cirrus.yml | 6 ++- .gitignore | 1 + Makefile | 93 ++++++++++++++++----------------------- ci/Containerfile | 1 + gcsupld/Containerfile | 3 +- imgobsolete/Containerfile | 3 +- imgprune/Containerfile | 3 +- imgts/Containerfile | 7 ++- lib.sh | 7 +-- orphanvms/Containerfile | 3 +- 10 files changed, 54 insertions(+), 73 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 9ed0b2be..8fe43ed9 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -98,12 +98,13 @@ container_images_task: &container_images TARGET_NAME: 'skopeo_cidev' DEST_FQIN: *fqin env: &image_env - TEMPDIR: '$CIRRUS_WORKING_DIR' # For quay.io/libpod namespace REG_USERNAME: ENCRYPTED[de755aef351c501ee480231c24eae25b15e2b2a2b7c629f477c1d427fc5269e360bb358a53bd8914605bae588e99b52a] REG_PASSWORD: ENCRYPTED[52268944bb0d6642c33efb1c5d7fb82d0c40f9e6988448de35827f9be2cc547c1383db13e8b21516dbd7a0a69a7ae536] script: ci/make_container_images.sh; - + package_cache: &package_cache + folder: "/tmp/automation_images_tmp/.cache/**" + fingerprint_key: "${TARGET_NAME}-cache-version-1" tooling_images_task: alias: tooling_images @@ -142,6 +143,7 @@ tooling_images_task: <<: *image_env PUSH_LATEST: 1 # scripts force to 0 if $CIRRUS_PR script: ci/make_container_images.sh; + package_cache: *package_cache base_images_task: diff --git a/.gitignore b/.gitignore index 7db10c53..328edacf 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ */*.json +/.cache diff --git a/Makefile b/Makefile index 79d62e3d..32ffdce6 100644 --- a/Makefile +++ b/Makefile @@ -90,18 +90,19 @@ ci_debug: $(_TEMPDIR)/ci_debug.tar ## Build and enter container for local develo -e TEMPDIR=$(_TEMPDIR) \ docker-archive:$< +# Takes 4 arguments: export filepath, FQIN, context dir, package cache key define podman_build podman build -t $(2) \ - -v $(_TEMPDIR)/var_cache_dnf:/var/cache/dnf:Z \ + -v $(_TEMPDIR)/.cache/$(4):/var/cache/dnf:Z \ + -v $(_TEMPDIR)/.cache/$(4):/var/cache/apt:Z \ --build-arg PACKER_VERSION=$(call err_if_empty,PACKER_VERSION) \ -f $(3)/Containerfile . rm -f $(1) podman save --quiet -o $(1) $(2) - podman rmi $(2) endef -$(_TEMPDIR)/ci_debug.tar: $(_TEMPDIR)/var_cache_dnf ci/Containerfile ci/install_packages.txt ci/install_packages.sh lib.sh - $(call podman_build,$@,ci_debug,ci) +$(_TEMPDIR)/ci_debug.tar: $(_TEMPDIR)/.cache/fedora ci/Containerfile ci/install_packages.txt ci/install_packages.sh lib.sh + $(call podman_build,$@,ci_debug,ci,fedora) $(_TEMPDIR): mkdir -p $@ @@ -109,7 +110,10 @@ $(_TEMPDIR): $(_TEMPDIR)/bin: $(_TEMPDIR) mkdir -p $@ -$(_TEMPDIR)/var_cache_dnf: $(_TEMPDIR) +$(_TEMPDIR)/.cache: $(_TEMPDIR) + mkdir -p $@ + +$(_TEMPDIR)/.cache/%: $(_TEMPDIR)/.cache mkdir -p $@ $(_TEMPDIR)/packer.zip: $(_TEMPDIR) @@ -181,8 +185,8 @@ image_builder_debug: $(_TEMPDIR)/image_builder_debug.tar ## Build and enter cont -e GAC_FILEPATH=$(call err_if_empty,GAC_FILEPATH) \ docker-archive:$< -$(_TEMPDIR)/image_builder_debug.tar: $(_TEMPDIR) $(_TEMPDIR)/var_cache_dnf image_builder/Containerfile image_builder/install_packages.txt ci/install_packages.sh lib.sh - $(call podman_build,$@,image_builder_debug,image_builder) +$(_TEMPDIR)/image_builder_debug.tar: $(_TEMPDIR)/.cache/centos image_builder/Containerfile image_builder/install_packages.txt ci/install_packages.sh lib.sh + $(call podman_build,$@,image_builder_debug,image_builder,centos) .PHONY: base_images # This needs to run in a virt/nested-virt capable environment @@ -205,75 +209,67 @@ endef .PHONY: fedora_podman fedora_podman: ## Build Fedora podman development container - $(call build_podman_container,$@) + $(call build_podman_container,$@,fedora) .PHONY: prior-fedora_podman prior-fedora_podman: ## Build Prior-Fedora podman development container - $(call build_podman_container,$@) + $(call build_podman_container,$@,prior-fedora) .PHONY: ubuntu_podman ubuntu_podman: ## Build Ubuntu podman development container - $(call build_podman_container,$@) + $(call build_podman_container,$@,ubuntu) -$(_TEMPDIR)/%_podman.tar: podman/Containerfile podman/setup.sh $(wildcard base_images/*.sh) $(wildcard cache_images/*.sh) $(_TEMPDIR) $(_TEMPDIR)/var_cache_dnf +$(_TEMPDIR)/%_podman.tar: podman/Containerfile podman/setup.sh $(wildcard base_images/*.sh) $(wildcard cache_images/*.sh) $(_TEMPDIR)/.cache/% podman build -t $*_podman:$(call err_if_empty,IMG_SFX) \ --build-arg=BASE_NAME=$(subst prior-,,$*) \ --build-arg=BASE_TAG=$(call err_if_empty,BASE_TAG) \ --build-arg=PACKER_BUILD_NAME=$(subst _podman,,$*) \ - -v $(_TEMPDIR)/var_cache_dnf:/var/cache/dnf:Z \ - -v $(_TEMPDIR)/var_cache_dnf:/var/cache/apt:Z \ + -v $(_TEMPDIR)/.cache/$*:/var/cache/dnf:Z \ + -v $(_TEMPDIR)/.cache/$*:/var/cache/apt:Z \ -f podman/Containerfile . rm -f $@ podman save --quiet -o $@ $*_podman:$(IMG_SFX) .PHONY: skopeo_cidev skopeo_cidev: $(_TEMPDIR)/skopeo_cidev.tar ## Build Skopeo development and CI container - -$(_TEMPDIR)/skopeo_cidev.tar: podman/fedora_release $(wildcard skopeo_base/*) $(_TEMPDIR) $(_TEMPDIR)/var_cache_dnf +$(_TEMPDIR)/skopeo_cidev.tar: podman/fedora_release $(wildcard skopeo_base/*) $(_TEMPDIR)/.cache/fedora podman build -t skopeo_cidev:$(call err_if_empty,IMG_SFX) \ --build-arg=BASE_TAG=$(_fedora_podman_release) \ - -v $(_TEMPDIR)/var_cache_dnf:/var/cache/dnf:Z \ + -v $(_TEMPDIR)/.cache/fedora:/var/cache/dnf:Z \ skopeo_cidev rm -f $@ podman save --quiet -o $@ skopeo_cidev:$(IMG_SFX) .PHONY: imgts imgts: $(_TEMPDIR)/imgts.tar ## Build the VM image time-stamping container image -$(_TEMPDIR)/imgts.tar: imgts/Containerfile imgts/entrypoint.sh imgts/google-cloud-sdk.repo imgts/lib_entrypoint.sh $(_TEMPDIR) - podman build -t imgts:$(call err_if_empty,IMG_SFX) \ - -f imgts/Containerfile . - rm -f $@ - podman save --quiet -o $@ imgts:$(IMG_SFX) +$(_TEMPDIR)/imgts.tar: imgts/Containerfile imgts/entrypoint.sh imgts/google-cloud-sdk.repo imgts/lib_entrypoint.sh $(_TEMPDIR)/.cache/centos + $(call podman_build,$@,imgts:$(call err_if_empty,IMG_SFX),imgts,centos) -.PHONY: imgobsolete -imgobsolete: $(_TEMPDIR)/imgobsolete.tar ## Build the VM Image obsoleting container image -$(_TEMPDIR)/imgobsolete.tar: $(_TEMPDIR)/imgts.tar imgts/lib_entrypoint.sh imgobsolete/Containerfile imgobsolete/entrypoint.sh $(_TEMPDIR) +define imgts_base_podman_build podman load -i $(_TEMPDIR)/imgts.tar podman tag imgts:$(call err_if_empty,IMG_SFX) imgts:latest - podman build -t imgobsolete:$(call err_if_empty,IMG_SFX) \ - -f imgobsolete/Containerfile . - rm -f $@ - podman save --quiet -o $@ imgobsolete:$(IMG_SFX) + $(call podman_build,$@,$(1):$(call err_if_empty,IMG_SFX),$(1),centos) +endef + +.PHONY: imgobsolete +imgobsolete: $(_TEMPDIR)/imgobsolete.tar ## Build the VM Image obsoleting container image +$(_TEMPDIR)/imgobsolete.tar: $(_TEMPDIR)/imgts.tar imgts/lib_entrypoint.sh imgobsolete/Containerfile imgobsolete/entrypoint.sh $(_TEMPDIR)/.cache/centos + $(call imgts_base_podman_build,imgobsolete) .PHONY: imgprune imgprune: $(_TEMPDIR)/imgprune.tar ## Build the VM Image pruning container image -$(_TEMPDIR)/imgprune.tar: $(_TEMPDIR)/imgts.tar imgts/lib_entrypoint.sh imgprune/Containerfile imgprune/entrypoint.sh $(_TEMPDIR) - podman load -i $(_TEMPDIR)/imgts.tar - podman tag imgts:$(call err_if_empty,IMG_SFX) imgts:latest - podman build -t imgprune:$(call err_if_empty,IMG_SFX) \ - -f imgprune/Containerfile . - rm -f $@ - podman save --quiet -o $@ imgprune:$(IMG_SFX) +$(_TEMPDIR)/imgprune.tar: $(_TEMPDIR)/imgts.tar imgts/lib_entrypoint.sh imgprune/Containerfile imgprune/entrypoint.sh $(_TEMPDIR)/.cache/centos + $(call imgts_base_podman_build,imgprune) .PHONY: gcsupld gcsupld: $(_TEMPDIR)/gcsupld.tar ## Build the GCS Upload container image -$(_TEMPDIR)/gcsupld.tar: $(_TEMPDIR)/imgts.tar imgts/lib_entrypoint.sh gcsupld/Containerfile gcsupld/entrypoint.sh $(_TEMPDIR) - podman load -i $(_TEMPDIR)/imgts.tar - podman tag imgts:$(call err_if_empty,IMG_SFX) imgts:latest - podman build -t gcsupld:$(call err_if_empty,IMG_SFX) \ - -f gcsupld/Containerfile . - rm -f $@ - podman save --quiet -o $@ gcsupld:$(IMG_SFX) +$(_TEMPDIR)/gcsupld.tar: $(_TEMPDIR)/imgts.tar imgts/lib_entrypoint.sh gcsupld/Containerfile gcsupld/entrypoint.sh $(_TEMPDIR)/.cache/centos + $(call imgts_base_podman_build,gcsupld) + +.PHONY: orphanvms +orphanvms: $(_TEMPDIR)/orphanvms.tar ## Build the Orphaned VM container image +$(_TEMPDIR)/orphanvms.tar: $(_TEMPDIR)/imgts.tar imgts/lib_entrypoint.sh orphanvms/Containerfile orphanvms/entrypoint.sh $(_TEMPDIR)/.cache/centos + $(call imgts_base_podman_build,orphanvms) .PHONY: .get_ci_vm get_ci_vm: $(_TEMPDIR)/get_ci_vm.tar ## Build the get_ci_vm container image @@ -282,20 +278,9 @@ $(_TEMPDIR)/get_ci_vm.tar: lib.sh get_ci_vm/Containerfile get_ci_vm/entrypoint.s rm -f $@ podman save --quiet -o $@ get_ci_vm:$(IMG_SFX) -.PHONY: orphanvms -orphanvms: $(_TEMPDIR)/orphanvms.tar ## Build the Orphaned VM container image -$(_TEMPDIR)/orphanvms.tar: $(_TEMPDIR)/imgts.tar imgts/lib_entrypoint.sh orphanvms/Containerfile orphanvms/entrypoint.sh $(_TEMPDIR) - podman load -i $(_TEMPDIR)/imgts.tar - podman tag imgts:$(call err_if_empty,IMG_SFX) imgts:latest - podman build -t orphanvms:$(IMG_SFX) \ - -f orphanvms/Containerfile . - rm -f $@ - podman save --quiet -o $@ orphanvms:$(IMG_SFX) - - .PHONY: clean clean: ## Remove all generated files referenced in this Makefile - -rm -vrf $(_TEMPDIR) + -rm -rf $(_TEMPDIR) -rm -f image_builder/*.json -rm -f base_images/{*.json,cidata*,*-data} -rm -f ci_debug.tar diff --git a/ci/Containerfile b/ci/Containerfile index 7eeb9353..09b8215f 100644 --- a/ci/Containerfile +++ b/ci/Containerfile @@ -27,6 +27,7 @@ RUN dnf update -y && \ gawk \ jq \ make \ + openssh-clients \ python3-pyyaml \ rootfiles \ rsync \ diff --git a/gcsupld/Containerfile b/gcsupld/Containerfile index 7652ef12..c64e8c02 100644 --- a/gcsupld/Containerfile +++ b/gcsupld/Containerfile @@ -1,7 +1,6 @@ FROM imgts:latest -RUN yum -y update && \ - yum clean all +RUN yum -y --setopt=keepcache=true update COPY /gcsupld/entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod 755 /usr/local/bin/entrypoint.sh diff --git a/imgobsolete/Containerfile b/imgobsolete/Containerfile index fc0b4ca5..769b1ae4 100644 --- a/imgobsolete/Containerfile +++ b/imgobsolete/Containerfile @@ -1,7 +1,6 @@ FROM imgts:latest -RUN yum -y update && \ - yum clean all +RUN yum -y --setopt=keepcache=true update COPY /imgobsolete/entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod 755 /usr/local/bin/entrypoint.sh diff --git a/imgprune/Containerfile b/imgprune/Containerfile index 454f226b..a92412b2 100644 --- a/imgprune/Containerfile +++ b/imgprune/Containerfile @@ -1,7 +1,6 @@ FROM imgts:latest -RUN yum -y update && \ - yum clean all +RUN yum -y --setopt=keepcache=true update COPY /imgprune/entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod 755 /usr/local/bin/entrypoint.sh diff --git a/imgts/Containerfile b/imgts/Containerfile index ed2a40c7..f3d4d977 100644 --- a/imgts/Containerfile +++ b/imgts/Containerfile @@ -2,10 +2,9 @@ FROM quay.io/centos/centos:stream8 # Only needed for installing build-time dependencies COPY /imgts/google-cloud-sdk.repo /etc/yum.repos.d/google-cloud-sdk.repo -RUN yum -y update && \ - yum -y install epel-release && \ - yum -y install google-cloud-sdk && \ - yum clean all +RUN yum -y --setopt=keepcache=true update && \ + yum -y --setopt=keepcache=true install epel-release && \ + yum -y --setopt=keepcache=true install google-cloud-sdk # These all represent required variables which must be set by caller ENV GCPJSON="__unknown__" \ diff --git a/lib.sh b/lib.sh index e513afd4..8cb9327f 100644 --- a/lib.sh +++ b/lib.sh @@ -174,8 +174,7 @@ common_finalize() { rh_finalize() { set +e # Don't fail at the very end set -x - $SUDO dnf clean all - $SUDO rm -rf /var/cache/{yum,dnf} + # Packaging cache is preserved across builds of container images $SUDO rm -f /etc/udev/rules.d/*-persistent-*.rules $SUDO touch /.unconfigured # force firstboot to run common_finalize @@ -185,9 +184,7 @@ rh_finalize() { ubuntu_finalize() { set +e # Don't fail at the very end set -x - $SUDO apt-get -qq -y autoremove - $SUDO rm -f /var/cache/apt/* # DON'T remove archives: packages downloaded there - $SUDO rm -rf /var/lib/apt/lists/* + # Packaging cache is preserved across builds of container images common_finalize } diff --git a/orphanvms/Containerfile b/orphanvms/Containerfile index 62404395..d7fb08d3 100644 --- a/orphanvms/Containerfile +++ b/orphanvms/Containerfile @@ -1,7 +1,6 @@ FROM imgts:latest -RUN yum -y update && \ - yum clean all +RUN yum -y --setopt=keepcache=true update COPY /orphanvms/entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod 755 /usr/local/bin/entrypoint.sh