diff --git a/Jenkinsfile b/Jenkinsfile index 54cfd28..816cd11 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -41,4 +41,4 @@ //@Library(value="pipeline-lib@your_branch") _ /* groovylint-disable-next-line CompileStatic */ -packageBuildingPipeline(['distros' : ['centos7', 'el8', 'el9', 'leap15']]) +packageBuildingPipeline(['distros' : ['el8', 'el9', 'leap15']]) diff --git a/README.md b/README.md index 6ddb91f..66f22c2 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,3 @@ -# DISCONTINUATION OF PROJECT # -This project will no longer be maintained by Intel. -Intel has ceased development and contributions including, but not limited to, maintenance, bug fixes, new releases, or updates, to this project. -Intel no longer accepts patches to this project. - If you have an ongoing need to use this project, are interested in independently developing it, or would like to maintain patches for the open source software community, please create your own fork of this project. - # Simul What is Simul? diff --git a/packaging/Dockerfile.centos.7 b/packaging/Dockerfile.centos.7 index cdfb7f6..189ea1e 100644 --- a/packaging/Dockerfile.centos.7 +++ b/packaging/Dockerfile.centos.7 @@ -5,9 +5,31 @@ # # Pull base image -FROM centos:7 +FROM centos:centos7 LABEL maintainer="daos@daos.groups.io" +# Use local repo server if present +ARG REPO_FILE_URL +RUN set -e; \ + if [ -n "$REPO_FILE_URL" ]; then \ + cd /etc/yum.repos.d/ && \ + curl -k -f -o daos_ci-centos7-artifactory.repo.tmp \ + "$REPO_FILE_URL"daos_ci-centos7-artifactory.repo && \ + for file in *.repo; do \ + true > $file; \ + done; \ + mv daos_ci-centos7-artifactory.repo{.tmp,}; \ + fi; \ + yum -y install dnf; \ + yum clean all; \ + dnf --disablerepo \*epel\* -y install epel-release \ + dnf-plugins-core; \ + if [ -n "$REPO_FILE_URL" ]; then \ + dnf -y --quiet config-manager --disable epel; \ + fi; \ + dnf -y update epel-release; \ + dnf -y clean all + # use same UID as host and default value of 1000 if not specified ARG UID=1000 @@ -15,9 +37,9 @@ ARG UID=1000 #Nothing to do for CentOS # Install basic tools -RUN yum install -y epel-release -RUN yum install -y mock make rpm-build curl createrepo rpmlint redhat-lsb-core \ - git python-srpm-macros dnf +RUN dnf install -y epel-release +RUN dnf install -y mock make rpm-build curl createrepo rpmlint redhat-lsb-core \ + git python-srpm-macros dnf && dnf -y clean all # Add build user (to keep rpmbuild happy) ENV USER build diff --git a/packaging/Dockerfile.coverity b/packaging/Dockerfile.coverity index 7eed2c3..f8171c9 100755 --- a/packaging/Dockerfile.coverity +++ b/packaging/Dockerfile.coverity @@ -1,12 +1,13 @@ # # Copyright 2018-2020, Intel Corporation +# Copyright 2025 Hewlett Packard Enterprise Development LP # # 'recipe' for Docker to build for a Coverity scan. # # Pull base image FROM fedora:latest -MAINTAINER daos-stack +LABEL maintainer="daos-stack "" # use same UID as host and default value of 1000 if not specified ARG UID=1000 diff --git a/packaging/Dockerfile.mockbuild b/packaging/Dockerfile.mockbuild index c8bc1a4..4d8f243 100644 --- a/packaging/Dockerfile.mockbuild +++ b/packaging/Dockerfile.mockbuild @@ -1,11 +1,12 @@ # -# Copyright 2018-2023 Intel Corporation +# Copyright 2018-2024 Intel Corporation +# Copyright 2025 Hewlett Packard Enterprise Development LP # # 'recipe' for Docker to build an RPM # # Pull base image -ARG FVERSION=38 +ARG FVERSION=latest FROM fedora:$FVERSION # Needed for later use of FVERSION ARG FVERSION @@ -13,29 +14,37 @@ LABEL maintainer="daos@daos.groups.io" # Use local repo server if present ARG REPO_FILE_URL -RUN if [ -n "$REPO_FILE_URL" ]; then \ - cd /etc/yum.repos.d/ && \ - curl -f -o daos_ci-fedora-artifactory.repo.tmp \ - "$REPO_FILE_URL"daos_ci-fedora-artifactory.repo && \ - rm -f *.repo && \ - mv daos_ci-fedora-artifactory.repo{.tmp,}; \ - fi +ARG DAOS_LAB_CA_FILE_URL +ARG REPOSITORY_NAME +# script to install OS updates basic tools and daos dependencies +# COPY ./utils/scripts/install-fedora.sh /tmp/install.sh +# script to setup local repo if available +COPY ./packaging/scripts/repo-helper-fedora.sh /tmp/repo-helper.sh + +RUN chmod +x /tmp/repo-helper.sh && \ + /tmp/repo-helper.sh && \ + rm -f /tmp/repo-helper.sh # Install basic tools -RUN dnf -y install mock make \ - rpm-build createrepo rpmlint redhat-lsb-core git \ - python-srpm-macros rpmdevtools +RUN dnf -y install mock make \ + rpm-build createrepo rpmlint git \ + python-srpm-macros rpmdevtools && \ + dnf -y clean all # use same UID as host and default value of 1000 if not specified ARG UID=1000 # Add build user (to keep rpmbuild happy) -ENV USER build -ENV PASSWD build -RUN useradd -u $UID -ms /bin/bash $USER -RUN echo "$USER:$PASSWD" | chpasswd +ENV USER=build +ENV PASSWD=build # add the user to the mock group so it can run mock -RUN usermod -a -G mock $USER +RUN if [ $UID != 0 ]; then \ + useradd -u $UID -ms /bin/bash $USER; \ + echo "$USER:$PASSWD" | chpasswd; \ + usermod -a -G mock $USER; \ + mkdir -p /var/cache/mock; \ + chown $USER:root /var/cache/mock; \ + fi ARG CB0 RUN dnf -y upgrade && \ @@ -45,15 +54,24 @@ RUN dnf -y upgrade && \ # https://github.com/rpm-software-management/rpmlint/pull/795 in it # But make sure to patch after dnf upgrade so that an upgraded rpmlint # RPM doesn't wipe out our patch -COPY packaging/rpmlint--ignore-unused-rpmlintrc.patch . +# Ditto for the patch to zero and display ccache stats +# https://github.com/rpm-software-management/mock/pull/1299 +ARG PACKAGINGDIR=packaging +COPY ${PACKAGINGDIR}/*.patch ./ RUN (cd $(python3 -c 'import site; print(site.getsitepackages()[-1])') && \ if ! grep -e --ignore-unused-rpmlintrc rpmlint/cli.py; then \ - if ! patch -p1; then \ + if ! patch -p1 < $OLDPWD/rpmlint--ignore-unused-rpmlintrc.patch; then \ exit 1; \ fi; \ rm -f rpmlint/__pycache__/{cli,lint}.*.pyc; \ - fi) < rpmlint--ignore-unused-rpmlintrc.patch; \ - rm -f rpmlint--ignore-unused-rpmlintrc.patch + fi; \ + if ! grep _ccachePostBuildHook mockbuild/plugins/ccache.py; then \ + if ! patch -p3 < $OLDPWD/ccache-stats.patch; then \ + exit 1; \ + fi; \ + rm -f mockbuild/plugins/__pycache__/ccache.*.pyc; \ + fi); \ + rm -f rpmlint--ignore-unused-rpmlintrc.patch ccache-stats.patch # show the release that was built ARG CACHEBUST diff --git a/packaging/Dockerfile.ubuntu b/packaging/Dockerfile.ubuntu new file mode 100644 index 0000000..1e3813d --- /dev/null +++ b/packaging/Dockerfile.ubuntu @@ -0,0 +1,51 @@ +# Keep Dockerfile.ubuntu the same as this file until all packaging +# jobs are fixed to have a Dockerfile.ubuntu, and then the common +# Jenkinsfile will be changed to use Dockerfile.ubuntu. +# +# Copyright 2019-2021, Intel Corporation +# Copyright 2025 Hewlett Packard Enterprise Development LP +# +# 'recipe' for Docker to build an Debian package +# +# Pull base image +ARG BASE_DISTRO=ubuntu:20.04 +FROM $BASE_DISTRO +LABEL org.opencontainers.image.authors="daos@daos.groups.io" +# Needed for later use of BASE_DISTRO +ARG BASE_DISTRO + +ARG REPO_FILE_URL +ARG DAOS_LAB_CA_FILE_URL +ARG REPOSITORY_NAME +# script to setup local repo if available +COPY ./scripts/repo-helper-ubuntu.sh /tmp/repo-helper.sh + +RUN chmod +x /tmp/repo-helper.sh && \ + /tmp/repo-helper.sh && \ + rm -f /tmp/repo-helper.sh + +# Install basic tools +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + autoconf bash ca-certificates curl debhelper dh-make \ + dpkg-dev dh-python doxygen gcc git git-buildpackage \ + javahelper locales make patch pbuilder pkg-config \ + python3-dev python3-distro python3-distutils rpm scons wget \ + cmake valgrind rpmdevtools + +# use same UID as host and default value of 1000 if not specified +ARG UID=1000 + +# Add build user (to keep chrootbuild happy) +ENV USER=build +RUN useradd -u $UID -ms /bin/bash $USER + +# need to run the build command as root, as it needs to chroot +RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then \ + echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \ + fi; \ + echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build; \ + chmod 0440 /etc/sudoers.d/build; \ + visudo -c; \ + sudo -l -U build diff --git a/packaging/Dockerfile.ubuntu.20.04 b/packaging/Dockerfile.ubuntu.20.04 index ec76bfd..bd5363b 100644 --- a/packaging/Dockerfile.ubuntu.20.04 +++ b/packaging/Dockerfile.ubuntu.20.04 @@ -1,57 +1,42 @@ +# Keep Dockerfile.ubuntu the same as this file until all packaging +# jobs are fixed to have a Dockerfile.ubuntu, and then the common +# Jenkinsfile will be changed to use Dockerfile.ubuntu. # # Copyright 2019-2021, Intel Corporation +# Copyright 2025 Hewlett Packard Enterprise Development LP # # 'recipe' for Docker to build an Debian package # # Pull base image -FROM ubuntu:20.04 +ARG BASE_DISTRO=ubuntu:20.04 +FROM $BASE_DISTRO LABEL org.opencontainers.image.authors="daos@daos.groups.io" - -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - curl gpg +# Needed for later use of BASE_DISTRO +ARG BASE_DISTRO ARG REPO_FILE_URL -RUN if [ -n "$REPO_FILE_URL" ]; then \ - cd /etc/apt/sources.list.d && \ - curl -f -o daos_ci-ubuntu20.04-artifactory.list.tmp \ - "$REPO_FILE_URL"daos_ci-ubuntu20.04-artifactory.list && \ - true > ../sources.list && \ - mv daos_ci-ubuntu20.04-artifactory.list.tmp \ - daos_ci-ubuntu20.04-artifactory.list; \ - fi; \ - cd -; \ - curl -f -O "$REPO_FILE_URL"esad_repo.key; \ - gpg --no-default-keyring --keyring ./temp-keyring.gpg \ - --import esad_repo.key; \ - mkdir -p /usr/local/share/keyrings/; \ - gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \ - --output /usr/local/share/keyrings/daos-stack-public.gpg; \ - rm ./temp-keyring.gpg; \ - url_prefix=https://downloads.linux.hpe.com/SDR/; \ - for url in hpPublicKey2048.pub \ - hpPublicKey2048_key1.pub \ - hpePublicKey2048_key1.pub; do \ - curl -f -O "$url_prefix$url"; \ - gpg --no-default-keyring --keyring ./temp-keyring.gpg \ - --import "$(basename $url)"; \ - done; \ - gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \ - --output /usr/local/share/keyrings/hpe-sdr-public.gpg; \ - rm ./temp-keyring.gpg +ARG DAOS_LAB_CA_FILE_URL +ARG REPOSITORY_NAME +# script to setup local repo if available +COPY ./packaging/scripts/repo-helper-ubuntu.sh /tmp/repo-helper.sh + +RUN chmod +x /tmp/repo-helper.sh && \ + /tmp/repo-helper.sh && \ + rm -f /tmp/repo-helper.sh -# Install basic tools +# Install basic tools - rpmdevtools temporary commented out. RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ autoconf bash ca-certificates curl debhelper dh-make \ dpkg-dev dh-python doxygen gcc git git-buildpackage \ javahelper locales make patch pbuilder pkg-config \ - python3-dev python3-distro python3-distutils rpm scons wget \ - cmake valgrind rpmdevtools + python3-dev python3-distro python3-distutils rpm scons sudo \ + wget cmake valgrind # rpmdevtools # use same UID as host and default value of 1000 if not specified ARG UID=1000 # Add build user (to keep chrootbuild happy) -ENV USER build +ENV USER=build RUN useradd -u $UID -ms /bin/bash $USER # need to run the build command as root, as it needs to chroot @@ -59,7 +44,7 @@ RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \ fi; \ echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build; \ - echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \ echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build; \ chmod 0440 /etc/sudoers.d/build; \ visudo -c; \ diff --git a/packaging/Dockerfile.ubuntu.rolling b/packaging/Dockerfile.ubuntu.rolling index 02aca45..ed5d363 100644 --- a/packaging/Dockerfile.ubuntu.rolling +++ b/packaging/Dockerfile.ubuntu.rolling @@ -1,11 +1,12 @@ # # Copyright 2019, Intel Corporation +# Copyright 2025 Hewlett Packard Enterprise Development LP # # 'recipe' for Docker to build an Debian package # # Pull base image FROM ubuntu:rolling -Maintainer daos-stack +LABEL org.opencontainers.image.authors="daos@daos.groups.io" # use same UID as host and default value of 1000 if not specified ARG UID=1000 diff --git a/packaging/Makefile_distro_vars.mk b/packaging/Makefile_distro_vars.mk index 6a7f88b..79ae211 100644 --- a/packaging/Makefile_distro_vars.mk +++ b/packaging/Makefile_distro_vars.mk @@ -83,6 +83,22 @@ DISTRO_VERSION ?= $(VERSION_ID) ORIG_TARGET_VER := 15.4 SED_EXPR := 1p endif +ifeq ($(CHROOT_NAME),opensuse-leap-15.5-x86_64) +VERSION_ID := 15.5 +DISTRO_ID := sl15.5 +DISTRO_BASE := LEAP_15 +DISTRO_VERSION ?= $(VERSION_ID) +ORIG_TARGET_VER := 15.5 +SED_EXPR := 1p +endif +ifeq ($(CHROOT_NAME),opensuse-leap-15.6-x86_64) +VERSION_ID := 15.6 +DISTRO_ID := sl15.6 +DISTRO_BASE := LEAP_15 +DISTRO_VERSION ?= $(VERSION_ID) +ORIG_TARGET_VER := 15.6 +SED_EXPR := 1p +endif endif ifeq ($(ID),centos) ID = el diff --git a/packaging/Makefile_packaging.mk b/packaging/Makefile_packaging.mk index 2fdcaed..75038c1 100644 --- a/packaging/Makefile_packaging.mk +++ b/packaging/Makefile_packaging.mk @@ -35,7 +35,11 @@ TEST_PACKAGES ?= ${NAME} # unfortunately we cannot always name the repo the same as the project REPO_NAME ?= $(NAME) +ifneq ($(CI_PR_REPOS),) +PR_REPOS ?= $(CI_PR_REPOS) +else PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos: *\(.*\)/\1/p') +endif LEAP_15_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-leap15: *\(.*\)/\1/p') EL_7_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el7: *\(.*\)/\1/p') EL_8_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el8: *\(.*\)/\1/p') @@ -54,7 +58,8 @@ RPM_BUILD_OPTIONS := $(BUILD_DEFINES) GIT_DIFF_EXCLUDES := $(PATCH_EXCLUDE_FILES:%=':!%') endif -COMMON_RPM_ARGS = --define "_topdir $$PWD/_topdir" $(BUILD_DEFINES) +FVERSION ?= latest +COMMON_RPM_ARGS := --define "_topdir $$PWD/_topdir" $(BUILD_DEFINES) SPEC := $(shell if [ -f $(NAME)-$(DISTRO_BASE).spec ]; then echo $(NAME)-$(DISTRO_BASE).spec; else echo $(NAME).spec; fi) VERSION = $(eval VERSION := $(shell rpm $(COMMON_RPM_ARGS) --specfile --qf '%{version}\n' $(SPEC) | sed -n '1p'))$(VERSION) DEB_RVERS := $(subst $(DOT),\$(DOT),$(VERSION)) @@ -363,12 +368,14 @@ chrootbuild: $(SRPM) $(CALLING_MAKEFILE) LOCAL_REPOS='$(LOCAL_REPOS)' \ ARTIFACTORY_URL="$(ARTIFACTORY_URL)" \ DISTRO_VERSION="$(DISTRO_VERSION)" \ + PACKAGE="$(NAME)" \ TARGET="$<" \ packaging/rpm_chrootbuild endif podman_chrootbuild: if ! podman build --build-arg REPO_FILE_URL=$(REPO_FILE_URL) \ + --build-arg FVERSION=$(FVERSION) \ -t $(subst +,-,$(CHROOT_NAME))-chrootbuild \ -f packaging/Dockerfile.mockbuild .; then \ echo "Container build failed"; \ @@ -386,7 +393,9 @@ podman_chrootbuild: exit 1; \ fi; \ rpmlint $$(ls /var/lib/mock/$(CHROOT_NAME)/result/*.rpm | \ - grep -v -e debuginfo -e debugsource -e src.rpm)' + grep -v -e debuginfo -e debugsource -e src.rpm)'; then \ + exit 1; \ + fi docker_chrootbuild: if ! $(DOCKER) build --build-arg UID=$$(id -u) -t chrootbuild \ @@ -419,6 +428,8 @@ packaging_check: --exclude libfabric.spec \ --exclude Makefile \ --exclude README.md \ + --exclude SECURITY.md \ + --exclude LICENSE \ --exclude _topdir \ --exclude \*.tar.\* \ --exclude \*.code-workspace \ diff --git a/packaging/ccache-stats.patch b/packaging/ccache-stats.patch new file mode 100644 index 0000000..26d5eeb --- /dev/null +++ b/packaging/ccache-stats.patch @@ -0,0 +1,66 @@ +From e87d916d7f49ea4949973adf0f09e9e5bf891e03 Mon Sep 17 00:00:00 2001 +From: "Brian J. Murrell" +Date: Tue, 30 Jan 2024 11:03:12 -0500 +Subject: [PATCH 1/2] Show ccache stats at the end of the build + +Zero the ccache stats at the beginning of the build and then display the +ccache stats at the end of the build to see how effective ccache was. + +Signed-off-by: Brian J. Murrell +--- + mock/py/mockbuild/plugins/ccache.py | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +diff --git a/mock/py/mockbuild/plugins/ccache.py b/mock/py/mockbuild/plugins/ccache.py +index 2666ad9fc..1080ffe68 100644 +--- a/mock/py/mockbuild/plugins/ccache.py ++++ b/mock/py/mockbuild/plugins/ccache.py +@@ -35,6 +35,7 @@ def __init__(self, plugins, conf, buildroot): + buildroot.preexisting_deps.append("ccache") + plugins.add_hook("prebuild", self._ccacheBuildHook) + plugins.add_hook("preinit", self._ccachePreInitHook) ++ plugins.add_hook("postbuild", self._ccachePostBuildHook) + buildroot.mounts.add( + BindMountPoint(srcpath=self.ccachePath, bindpath=buildroot.make_chroot_path("/var/tmp/ccache"))) + +@@ -47,6 +48,9 @@ def __init__(self, plugins, conf, buildroot): + @traceLog() + def _ccacheBuildHook(self): + self.buildroot.doChroot(["ccache", "-M", str(self.ccache_opts['max_cache_size'])], shell=False) ++ # zero ccache stats ++ getLog().info("Zero ccache stats:") ++ self.buildroot.doChroot(["ccache", "--zero-stats"], printOutput=True, shell=False) + + # set up the ccache dir. + # we also set a few variables used by ccache to find the shared cache. +@@ -61,3 +65,10 @@ def _ccachePreInitHook(self): + file_util.mkdirIfAbsent(self.buildroot.make_chroot_path('/var/tmp/ccache')) + file_util.mkdirIfAbsent(self.ccachePath) + self.buildroot.uid_manager.changeOwner(self.ccachePath, recursive=True) ++ ++ # get some cache stats ++ def _ccachePostBuildHook(self): ++ # show the cache hit stats ++ getLog().info("ccache stats:") ++ self.buildroot.doChroot(["ccache", "--show-stats"], printOutput=True, shell=False) +++ + +From bfd3a7e1bb47d28ee60a94cb5985c1f66476475f Mon Sep 17 00:00:00 2001 +From: "Brian J. Murrell" +Date: Tue, 30 Jan 2024 11:17:48 -0500 +Subject: [PATCH 2/2] Remove extraneous line + +Signed-off-by: Brian J. Murrell +--- + mock/py/mockbuild/plugins/ccache.py | 1 - + 1 file changed, 1 deletion(-) + +diff --git a/mock/py/mockbuild/plugins/ccache.py b/mock/py/mockbuild/plugins/ccache.py +index 1080ffe68..1a20846d3 100644 +--- a/mock/py/mockbuild/plugins/ccache.py ++++ b/mock/py/mockbuild/plugins/ccache.py +@@ -71,4 +71,3 @@ def _ccachePostBuildHook(self): + # show the cache hit stats + getLog().info("ccache stats:") + self.buildroot.doChroot(["ccache", "--show-stats"], printOutput=True, shell=False) +-+ diff --git a/packaging/debian_chrootbuild b/packaging/debian_chrootbuild index cc2cc96..5cb2943 100755 --- a/packaging/debian_chrootbuild +++ b/packaging/debian_chrootbuild @@ -2,17 +2,35 @@ set -uex +: "${REPO_FILE_URL:=}" +: "${HTTPS_PROXY:=}" + +# Currently not fully working behind a proxy if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then - echo "MIRRORSITE=${ARTIFACTORY_URL}artifactory/ubuntu-proxy" | sudo tee /root/.pbuilderrc + pbuilderrc="./pbuilder_rc.txt" + rm -f "$pbuilderrc" + if [ -n "${HTTPS_PROXY}" ]; then + echo "export http_proxy=\"${HTTPS_PROXY}\"" >> "$pbuilderrc" + else + echo "MIRRORSITE=${ARTIFACTORY_URL}/ubuntu-proxy/ubuntu" > "$pbuilderrc" + fi + #if [ -n "$REPO_FILE_URL" ]; then + # direct="${REPO_FILE_URL##*//}" + # direct="${direct%%/*}" + # echo "no_proxy=\"${direct}\"" >> "$pbuilderrc" + #fi + # shellcheck disable=SC2002 + cat "$pbuilderrc" | sudo tee /root/.pbuilderrc fi # shellcheck disable=SC2086 sudo pbuilder create \ --extrapackages "gnupg ca-certificates" \ - $DISTRO_ID_OPT + $DISTRO_ID_OPT || true # Ignore error status for now. repo_args="" repos_added=() +# currently a bit broken, pbuilder will not accept user provided CAs. for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do branch="master" build_number="lastSuccessfulBuild" @@ -32,31 +50,34 @@ for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do repo_args="$repo_args|deb [trusted=yes] ${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/$branch/$build_number/artifact/artifacts/$DISTRO/ ./" done -repo_args+="|$(curl -sSf "$REPO_FILE_URL"daos_ci-"$DISTRO"-artifactory.list | - sed -e 's/#.*//' -e '/ubuntu-proxy/d' -e '/^$/d' -e '/^$/d' \ - -e 's/signed-by=.*\.gpg/trusted=yes/' | - sed -e ':a; N; $!ba; s/\n/|/g')" -for repo in $JOB_REPOS; do - repo_name=${repo##*://} - repo_name=${repo_name//\//_} - if [[ " ${repos_added[*]} " = *\ ${repo_name}\ * ]]; then - # don't add duplicates, first found wins - continue - fi - repos_added+=("$repo_name") - repo_args+="|deb ${repo} $VERSION_CODENAME main" -done -# NB: This PPA is needed to support modern go toolchains on ubuntu 20.04. -# After the build is updated to use 22.04, which supports go >= 1.18, it -# should no longer be needed. -repo_args="$repo_args|deb [trusted=yes] https://ppa.launchpadcontent.net/longsleep/golang-backports/ubuntu $VERSION_CODENAME main" -echo "$repo_args" -if [ "$repo_args" = "|" ]; then - repo_args="" -else - #repo_args="--othermirror"${repo_args#|}\"" - repo_args="${repo_args#|}" -fi +# currently broken, builder will not accept internal certs. +# repo_args+="|$(curl -sSf "$REPO_FILE_URL"daos_ci-"$DISTRO"-artifactory.list | +# sed -e 's/#.*//' -e '/ubuntu-proxy/d' -e '/^$/d' -e '/^$/d' \ +# -e 's/signed-by=.*\.gpg/trusted=yes/' | +# sed -e ':a; N; $!ba; s/\n/|/g')" +#for repo in $JOB_REPOS; do +# repo_name=${repo##*://} +# repo_name=${repo_name//\//_} +# if [[ " ${repos_added[*]} " = *\ ${repo_name}\ * ]]; then +# # don't add duplicates, first found wins +# continue +# fi +# repos_added+=("$repo_name") +# repo_args+="|deb ${repo} $VERSION_CODENAME main" +#done + +## NB: This PPA is needed to support modern go toolchains on ubuntu 20.04. +## After the build is updated to use 22.04, which supports go >= 1.18, it +## should no longer be needed. +# currently broken - claim is public key not available. +#repo_args="$repo_args|deb [trusted=yes] https://ppa.launchpadcontent.net/longsleep/golang-backports/ubuntu $VERSION_CODENAME main" +#echo "$repo_args" +#if [ "$repo_args" = "|" ]; then +# repo_args="" +#else +# #repo_args="--othermirror"${repo_args#|}\"" +# repo_args="${repo_args#|}" +#fi cd "$DEB_TOP" # shellcheck disable=SC2086 sudo pbuilder update --override-config $DISTRO_ID_OPT ${repo_args:+--othermirror "$repo_args"} diff --git a/packaging/get_base_branch b/packaging/get_base_branch new file mode 100755 index 0000000..75eb90b --- /dev/null +++ b/packaging/get_base_branch @@ -0,0 +1,22 @@ +#!/bin/bash + +# find the base branch of the current branch + +set -eux -o pipefail +IFS=' ' read -r -a add_bases <<< "${1:-}" +origin="${ORIGIN:-origin}" +mapfile -t all_bases < <(echo "master" + git branch -r | sed -ne "/^ $origin\\/release\\/[0-9]/s/^ $origin\\///p") +all_bases+=("${add_bases[@]}") +TARGET="master" +min_diff=-1 +for base in "${all_bases[@]}"; do + git rev-parse --verify "$origin/$base" &> /dev/null || continue + commits_ahead=$(git log --oneline "$origin/$base..HEAD" | wc -l) + if [ "$min_diff" -eq -1 ] || [ "$min_diff" -gt "$commits_ahead" ]; then + TARGET="$base" + min_diff=$commits_ahead + fi +done +echo "$TARGET" +exit 0 diff --git a/packaging/rpm_chrootbuild b/packaging/rpm_chrootbuild index d122e0e..8d37abc 100755 --- a/packaging/rpm_chrootbuild +++ b/packaging/rpm_chrootbuild @@ -2,12 +2,34 @@ set -uex -original_cfg_file="/etc/mock/$CHROOT_NAME.cfg" -cfg_file=mock.cfg +: "${HTTPS_PROXY:=}" +: "${REPO_FILE_URL:=}" +: "${ARCH:=$(arch)}" +: "${REPOSITORY_NAME:=artifactory}" -cp "$original_cfg_file" "$cfg_file" +cp /etc/mock/"$CHROOT_NAME".cfg mock.cfg -if [[ $CHROOT_NAME == *epel-8-x86_64 ]]; then +# Enable mock ccache plugin +cat <> mock.cfg +config_opts['plugin_conf']['ccache_enable'] = True +config_opts['plugin_conf']['ccache_opts']['dir'] = "%(cache_topdir)s/%(root)s/ccache/" +EOF + + +# Optionally add a proxy to mock +if [ -n "$HTTPS_PROXY" ];then + yum_proxy="http://${HTTPS_PROXY##*//}" + echo "config_opts['https_proxy'] = '$yum_proxy'" >> mock.cfg +fi + +# No proxy for local mirrors +if [ -n "$REPO_FILE_URL" ]; then + direct="${REPO_FILE_URL##*//}" + direct="${direct%%/*}" + echo "config_opts['no_proxy'] = '${direct}'" >> mock.cfg +fi + +if [[ $CHROOT_NAME == *"epel-8-${ARCH}" ]]; then cat <> mock.cfg config_opts['module_setup_commands'] = [ ('enable', 'javapackages-tools:201801'), @@ -17,12 +39,12 @@ EOF fi # Use dnf on CentOS 7 -if [[ $CHROOT_NAME == *epel-7-x86_64 ]]; then +if [[ $CHROOT_NAME == *"epel-7-$ARCH" ]]; then MOCK_OPTIONS="--dnf --no-bootstrap-chroot${MOCK_OPTIONS:+ }$MOCK_OPTIONS" fi # Allow BR: foo-devel < 1.2 to work when foo-devel-1.3 is actually available -cat <> "$cfg_file" +cat <> mock.cfg config_opts['dnf.conf'] += """ [main] best=0 @@ -33,7 +55,7 @@ EOF repo_adds=() repo_dels=() -echo -e "config_opts['yum.conf'] += \"\"\"\n" >> "$cfg_file" +echo -e "config_opts['yum.conf'] += \"\"\"\n" >> mock.cfg if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then repo_dels+=("--disablerepo=\*") @@ -56,8 +78,8 @@ if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then REPO_FILE_URL="file://$(readlink -e "$REPO_FILES_PR")/" fi fi - curl -sSf "${REPO_FILE_URL}daos_ci-$DISTRO"-mock-artifactory.repo >> "$cfg_file" - repo_adds+=("--enablerepo *-artifactory") + curl -sSf "$REPO_FILE_URL"daos_ci-"${CHROOT_NAME%-*}".repo >> mock.cfg + repo_adds+=("--enablerepo *-${REPOSITORY_NAME}") fi fi @@ -81,9 +103,9 @@ for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do repo_adds+=("--enablerepo $repo:${branch//[@\/]/_}:$build_number") echo -e "[$repo:${branch//[@\/]/_}:$build_number]\n\ name=$repo:${branch//[@\/]/_}:$build_number\n\ -baseurl=${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/${branch//\//%2F}/$build_number/artifact/artifacts/$DISTRO/\n\ +baseurl=${ARTIFACTS_URL:-${JENKINS_URL:-https://build.hpdd.intel.com/}job/}daos-stack/job/$repo/job/${branch//\//%2F}/$build_number/artifact/artifacts/$DISTRO/\n\ enabled=1\n\ -gpgcheck=False\n" >> "$cfg_file" +gpgcheck=False\n" >> mock.cfg done for repo in $JOB_REPOS; do repo_name=${repo##*://} @@ -97,29 +119,46 @@ for repo in $JOB_REPOS; do echo -e "[${repo_name//[@\/]/_}]\n\ name=${repo_name}\n\ baseurl=${repo//\//%2F}\n\ -enabled=1\n" >> "$cfg_file" +enabled=1\n" >> mock.cfg done -echo "\"\"\"" >> "$cfg_file" +echo "\"\"\"" >> mock.cfg if [ -n "$DISTRO_VERSION" ]; then releasever_opt=("--config-opts=releasever=$DISTRO_VERSION") fi -bs_dir=/scratch/mock/cache/"${CHROOT_NAME}"-bootstrap -if ls -l /scratch/mock/cache/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz; then - mkdir -p "/var/cache/mock/${CHROOT_NAME}-bootstrap" +bs_dir=/scratch/mock/cache/"${CHROOT_NAME}"-bootstrap-$(id -u) +if ls -l "$bs_dir"/root_cache/cache.tar.gz; then + mkdir -p "/var/cache/mock/${CHROOT_NAME}-bootstrap/" flock "$bs_dir" -c "cp -a $bs_dir/root_cache /var/cache/mock/${CHROOT_NAME}-bootstrap" fi +if ls -l "$bs_dir/ccache-$CHROOT_NAME-$PACKAGE".tar.gz; then + flock "$bs_dir" -c "tar -C / -xzf $bs_dir/ccache-$CHROOT_NAME-$PACKAGE.tar.gz" +fi -# shellcheck disable=SC2086 -eval mock -r "$cfg_file" ${repo_dels[*]} ${repo_adds[*]} --disablerepo=\*-debug* \ - "${releasever_opt[@]}" $MOCK_OPTIONS $RPM_BUILD_OPTIONS "$TARGET" +rc=0 +# shellcheck disable=SC2086,SC2048 +if ! eval time mock -r mock.cfg ${repo_dels[*]} ${repo_adds[*]} --no-clean \ + --disablerepo=\*-debug* ${releasever_opt[*]} $MOCK_OPTIONS \ + $RPM_BUILD_OPTIONS "$TARGET"; then + rc=${PIPESTATUS[0]} +fi -date -if ls -l /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz && - [ -d /scratch/ ]; then - mkdir -p /scratch/mock/cache/"${CHROOT_NAME}"-bootstrap/ - if ! cmp /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz "$bs_dir"/root_cache/cache.tar.gz; then - flock "$bs_dir" -c "cp -a /var/cache/mock/${CHROOT_NAME}-bootstrap/root_cache $bs_dir/" +# Save the ccache +if [ -d /scratch/mock ]; then + mkdir -p "$bs_dir"/ + if ! flock "$bs_dir" -c "tar -czf $bs_dir/ccache-$CHROOT_NAME-$PACKAGE.tar.gz /var/cache/mock/${CHROOT_NAME}/ccache"; then + echo "Failed to save ccache. Plowing onward." + echo "I am $(id)" + fi + if ls -l /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz; then + if ! cmp /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz "$bs_dir"/root_cache/cache.tar.gz; then + if ! flock "$bs_dir" -c "cp -a /var/cache/mock/${CHROOT_NAME}-bootstrap/root_cache $bs_dir/"; then + echo "Failed to save root_cache. Plowing onward." + echo "I am $(id)" + fi + fi fi fi + +exit "$rc" diff --git a/packaging/scripts/repo-helper-fedora.sh b/packaging/scripts/repo-helper-fedora.sh new file mode 100644 index 0000000..6c8375e --- /dev/null +++ b/packaging/scripts/repo-helper-fedora.sh @@ -0,0 +1,79 @@ +#!/bin/bash +set -uex + +# This script is used by Dockerfiles to optionally use +# a local repository instead of a distro provided repository. + +: "${REPO_FILE_URL:=}" +: "${DAOS_LAB_CA_FILE_URL:=}" +: "${FVERSION:=latest}" +: "${REPOSITORY_NAME:=artifactory}" +: "${archive:=}" +if [ "$FVERSION" != "latest" ]; then + if [ "$FVERSION" != "42" ]; then + if [ "$FVERSION" != "41" ]; then + archive="-archive" + fi + fi +fi + +# shellcheck disable=SC2120 +disable_repos () { + local repos_dir="$1" + shift + local save_repos + IFS=" " read -r -a save_repos <<< "${*:-} daos_ci-fedora${archive}-${REPOSITORY_NAME}" + if [ -n "$REPO_FILE_URL" ]; then + pushd "$repos_dir" + local repo + for repo in "${save_repos[@]}"; do + mv "$repo".repo{,.tmp} + done + for file in *.repo; do + true > "$file" + done + for repo in "${save_repos[@]}"; do + mv "$repo".repo{.tmp,} + done + popd + fi +} + +# Use local repo server if present +install_curl() { + : +} + +# Use local repo server if present +install_optional_ca() { + ca_storage="/etc/pki/ca-trust/source/anchors/" + if [ -n "$DAOS_LAB_CA_FILE_URL" ]; then + curl -k --noproxy '*' -sSf -o "${ca_storage}lab_ca_file.crt" \ + "$DAOS_LAB_CA_FILE_URL" + update-ca-trust + fi +} + +# Use local repo server if present +# if a local repo server is present and the distro repo server can not +# be reached, have to bootstrap in an environment to get curl installed +# to then install the pre-built repo file. + +if [ -n "$REPO_FILE_URL" ]; then + install_curl + install_optional_ca + mkdir -p /etc/yum.repos.d + pushd /etc/yum.repos.d/ + curl -k --noproxy '*' -sSf \ + -o "daos_ci-fedora${archive}-${REPOSITORY_NAME}.repo" \ + "${REPO_FILE_URL}daos_ci-fedora${archive}-${REPOSITORY_NAME}.repo" + disable_repos /etc/yum.repos.d/ + popd +fi +dnf -y install dnf-plugins-core +# This does not work in fedora/41 anymore -- needs investigation +# dnf -y config-manager --save --setopt=assumeyes=True +# dnf config-manager --save --setopt=install_weak_deps=False +dnf clean all + +disable_repos /etc/yum.repos.d/ "${save_repos[@]}" diff --git a/packaging/scripts/repo-helper-ubuntu.sh b/packaging/scripts/repo-helper-ubuntu.sh new file mode 100644 index 0000000..20220e9 --- /dev/null +++ b/packaging/scripts/repo-helper-ubuntu.sh @@ -0,0 +1,103 @@ +#!/bin/bash +set -uex + +# This script is used by Dockerfiles to optionally use +# a local repository instead of a distro provided repository. +# It will also optionally allow running a /tmp/install script +# for custom packages if present. + +: "${REPO_FILE_URL:=}" +: "${HTTPS_PROXY:=}" +: "${DAOS_LAB_CA_FILE_URL:=}" +: "${REPOSITORY_NAME:=artifactory}" + +disable_repos () { + if [ -e /etc/apt/sources.list.d/ubuntu.sources ];then + mv /etc/apt/sources.list.d/ubuntu.sources \ + etc/apt/sources.list.d/ubuntu.sources.disabled + elif [ -e /etc/apt/sources.list ];then + mv /etc/apt/sources.list \ + etc/apt/sources.list.disabled + fi +} + +# Use local repo server if present +install_curl() { + + if command -v curl; then + echo "found curl!" + return + else + apt-get update + apt-get install curl ca-certificates gpg gpg-agent \ + software-properties-common + fi + + if command -v wget; then + echo "found wget!" + return + fi + # If we don't find one of these, we are basically sunk for using + # a local repository mirror. +} + +# Use local repo server if present +install_optional_ca() { + ca_storage="/usr/local/share/ca-certificates/" + if [ -n "$DAOS_LAB_CA_FILE_URL" ]; then + curl -k --noproxy '*' -sSf -o "${ca_storage}lab_ca_file.crt" \ + "$DAOS_LAB_CA_FILE_URL" + update-ca-certificates + fi +} + +echo "APT::Get::Assume-Yes \"true\";" > /etc/apt/apt.conf.d/no-prompt +echo "APT::Install-Recommends \"false\";" > /etc/apt/apt.conf.d/no-recommends +if [ -n "$HTTPS_PROXY" ];then + apt_proxy="http://${HTTPS_PROXY##*//}" + echo "Acquire::http::Proxy \"$apt_proxy\";" > \ + /etc/apt/apt.conf.d/local_proxy + if [ -n "$REPO_FILE_URL" ]; then + direct="${REPO_FILE_URL##*//}" + direct="${direct%%/*}" + echo "Acquire::http::Proxy { $direct DIRECT; };" >> \ + /etc/apt/apt.conf.d/local_proxy + fi +fi + +# Use local repo server if present +# if a local repo server is present and the distro repo server can not +# be reached, have to bootstrap in an environment to get curl installed +# to then install the pre-built repo file. +DISTRO_VERSION="${BASE_DISTRO##*:}" +if [ -n "$REPO_FILE_URL" ]; then + install_curl + install_optional_ca + # Ubuntu local repo mirror is not working + # curl -k --noproxy '*' -sSf \ + # -o "daos_ci-ubuntu${DISTRO_VERSION}-${REPOSITORY_NAME}.list" \ + # "${REPO_FILE_URL}daos_ci-ubuntu${DISTRO_VERSION}-${REPOSITORY_NAME}.list" + # disable_repos + # temp hack until we can debug the Ubuntu repos. + REPO_BASE="${REPO_FILE_URL%repo-files/}" + curl -k --noproxy '*' -sSf \ + -o "rpmdevtools_8.10-10_amd64.deb" \ + "${REPO_BASE}daos-stack-deps-ubuntu-${DISTRO_VERSION}-x86_64-stable-local/pool/rpmdevtools_8.10-10_amd64.deb" + apt-get install "./rpmdevtools_8.10-10_amd64.deb" + mkdir -p /usr/local/share/keyrings/ + curl --noproxy '*' -sSf -O "${REPO_FILE_URL}esad_repo.key" + gpg --no-default-keyring --keyring ./temp-keyring.gpg \ + --import esad_repo.key + gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \ + --output /usr/local/share/keyrings/daos-stack-public.gpg +fi + +apt-get update +apt-get upgrade +# add-apt-repository ppa:longsleep/golang-backports +apt-get update +if [ -e /tmp/install.sh ]; then + chmod +x /tmp/install.sh + /tmp/install.sh +fi +apt-get clean all