build-anvil-ext-repo

#!/bin/bash

set -e

function print_usage() {
	echo "$(basename $0)"
	echo ""
	echo "usage:"
	echo ""
	echo "$(basename $0) [options]"
	echo ""
	echo "Options:"
	echo " -h help"
	echo " -d distribution          centos-8-stream|rhel-8|almalinux-9|rhel-9 (default: centos-8-stream)"
	echo " -l                       list currently supported distro"
	echo " -w                       wipe all mock local caches"
	echo " -r                       wipe srpms and temporary repositories before and after build"
	echo " -p path/to/publish_area  publish the newly built repo to path and make it latest"
	echo " -s                       skip rebuilding srpms (debug only)"
	echo " -c                       skip refreshing chroot (debug only)"
	echo " -b                       skip build (debug only)"
	echo ""
}

function list_distros() {
	cd configs
	distros=$(ls -1 | sed -e 's#-mock.*##' -e 's#-package.*##g' | sort -u)
	for i in $distros; do
		# print distros that have all required config files
		if [ -f ${i}-mock-build.cfg.in ] && \
		   [ -f ${i}-mock-test.cfg.in ] && \
		   [ -f ${i}-package-list ]; then
			echo $i
		fi
	done
}

# required tools to build archive and stuff
tools="mock wget rpmbuild createrepo"

# set defaults
distro=centos-8-stream
wipemock=0
wiperpm=0
skipsrpms=0
skipchroot=0
skipbuild=0
publish=""

while getopts ":hd:lwrscbp:" optflags; do
	case "$optflags" in
		b)
			skipbuild=1
			;;
		c)
			skipchroot=1
			;;
		d)
			distro="$OPTARG"
			;;
		l)
			list_distros
			exit 0
			;;
		p)
			publish="$OPTARG"
			;;
		r)
			wiperpm=1
			;;
		s)
			skipsrpms=1
			;;
		w)
			wipemock=1
			;;
		h)
			print_usage
			exit 0
			;;
		\?|:)
			print_usage
			exit 1
			;;
	esac
done

function check_requirements() {
	for i in $tools; do
		if [ -z "$(which $i 2>/dev/null)" ]; then
			echo "Error! $i missing from PATH"
			exit 1
		fi
	done
}

function check_distro() {
	for i in ${distro}-mock-build.cfg.in ${distro}-mock-test.cfg.in ${distro}-package-list; do
		if [ ! -f "configs/$i" ]; then
			echo "Error! Distribution ${distro} is currently missing configs/$i. Please add related config files"
			exit 1
		fi
	done
}

# functions below here can write files that needs to be removed

# set common variables

toplevel=$(pwd)
mockbuildconfig=$(pwd)/build/${distro}/mock-build.cfg
mocktestconfig=$(pwd)/build/${distro}/mock-test.cfg
localmockrepo=$(pwd)/build/${distro}/mockrepo
localsrpms=$(pwd)/build/${distro}/srpms
localmirror=$(pwd)/build/${distro}/mirror
tmpsrpms=$(pwd)/build/${distro}/tmpsrpms
anvilsrpms=$(pwd)/srpms
finalrepo=$(pwd)/build/${distro}/finalrepo

function cleanup() {
	if [ "$wiperpm" = "1" ]; then
		rm -rf ${localmockrepo}
		rm -rf ${localsrpms}
	fi
	if [ "$wipemock" = "1" ]; then
		if [ -f $mockbuildconfig ]; then
			mock --scrub=all -r $mockbuildconfig
			rm -rf $mockbuildconfig
		fi
		if [ -f $mocktestconfig ]; then
			mock --scrub=all -r $mocktestconfig
			rm -rf $mocktestconfig
		fi
	fi
	if [ -n "$1" ]; then
		exit $1
	fi
}

function create_dirs() {
	mkdir -p $localmockrepo
	mkdir -p $localsrpms
	mkdir -p $finalrepo
}

function prep_mock_configs() {
	cat configs/${distro}-mock-build.cfg.in | \
		sed -e 's#@LOCALREPO@#'$localmockrepo'/results/mock-build#g' \
		> ${mockbuildconfig}
	cat configs/${distro}-mock-test.cfg.in | \
		sed -e 's#@FINALREPO@#'$finalrepo'/rpms#g' \
		> ${mocktestconfig}
}

function init_mock() {
	mockconfig="$1"
	if [ "$skipchroot" = "1" ]; then
		return
	fi
	if [ "$wipemock" = "1" ]; then
		mock --scrub=all -r $mockconfig
	else
		mock --scrub=chroot -r $mockconfig
	fi
	mock --init -r $mockconfig
	mock --update -r $mockconfig
}

function download_srpms() {
	srpms="$@"
	echo "Downloading $srpms"
	mkdir -p $localmirror
	cd $localmirror
	cat > $localmirror/fedora-src.repo << EOF
[anvil-fedora-source]
name=Fedora \$releasever - Source
metalink=https://mirrors.fedoraproject.org/metalink?repo=fedora-source-\$releasever&arch=\$basearch
enabled=1
repo_gpgcheck=0
type=rpm
gpgcheck=0
skip_if_unavailable=False

[anvil-updates-source]
name=Fedora \$releasever - Updates Source
metalink=https://mirrors.fedoraproject.org/metalink?repo=updates-released-source-f\$releasever&arch=\$basearch
enabled=1
repo_gpgcheck=0
type=rpm
gpgcheck=0
skip_if_unavailable=False

[anvil-fedora-modular-source]
name=Fedora Modular \$releasever - Source
metalink=https://mirrors.fedoraproject.org/metalink?repo=fedora-modular-source-\$releasever&arch=\$basearch
enabled=1
repo_gpgcheck=0
type=rpm
gpgcheck=0
skip_if_unavailable=True

[anvil-updates-modular-source]
name=Fedora Modular \$releasever - Updates Source
metalink=https://mirrors.fedoraproject.org/metalink?repo=updates-released-modular-source-f\$releasever&arch=\$basearch
enabled=1
repo_gpgcheck=0
type=rpm
gpgcheck=0
skip_if_unavailable=True
EOF
	case $distro in
		centos-8*|rhel-8*)
			fedrelease=37
			;;
		*)
			fedrelease=40
			;;
	esac
	dnf download --source \
		--config $localmirror/fedora-src.repo \
		--releasever $fedrelease \
		--disablerepo="*" \
		--enablerepo=anvil-fedora-source --enablerepo=anvil-updates-source \
		--enablerepo=anvil-fedora-modular-source --enablerepo=anvil-updates-modular-source \
		--installroot=$localmirror/installroot \
		$srpms
	mv $localmirror/*src.rpm $localsrpms/
	cd $toplevel
	rm -rf $localmirror
}

# generate all srpms based on the list provided by config/${distro}-package-list
# if a package exists in srpms/ it will be given higher priority than
# downloading the sources from fedora via fedpkg
# packages in srpms/$PACKAGENAME must provide a build.sh script to
# generate the srpm

function prep_srpms() {
	if [ "$skipsrpms" = "1" ]; then
		return
	fi
	cd $localsrpms
	# force rebuild of drbd-kmod and perl-Log-Journald for test update repo
	rm -rf drbd-kmod-* perl-Log-Journald*
	for i in $packagelist; do
		# check if local override exists
		if echo $overridesrpms | grep -q $i; then
			echo "Local override for $i requested"
			overridedir=""
			if [ -d "$anvilsrpms/${distro}/$i" ]; then
				overridedir="$anvilsrpms/${distro}/$i"
			elif [ -d "$anvilsrpms/$i" ]; then
				overridedir="$anvilsrpms/$i"
			fi
			if [ -n "$overridedir" ]; then
				echo "Local override for $i detected ($overridedir)"
				cd "$overridedir"
				./build.sh
				for x in *.src.rpm; do
					if [ ! -f $localsrpms/$x ]; then
						mv $x $localsrpms/
					fi
				done
			else
				echo "Local override for $i not found!"
				exit 1
			fi
		else
			downloads="$downloads $i"
		fi
		cd $toplevel
	done
	cd $toplevel
	download_srpms "$downloads"
}

# concept of chain build is that we will continue to try builds
# till either all packages have built or stop with error
# after $number_of_package + 5 attemts

function build_rpms() {
	if [ "$skipbuild" = "1" ]; then
		return
	fi

	# force rebuild of drbd-kmod and perl-Log-Journald for test update repo
	rm -rf $localmockrepo/results/mock-build/drbd-kmod-*
	rm -rf $localmockrepo/results/mock-build/perl-Log-Journald*

	# add extra attempts in case we have network issues
	count=5
	for i in $packagelist; do
		count=$((count + 1))
	done

	current=0
	while [ $current -le $count ]; do
		if mock --enable-network -r $mockbuildconfig --localrepo=$localmockrepo --continue --chain $localsrpms/*.src.rpm; then
			echo "Built complited in $current loops"
			return
		fi
		current=$((current + 1))
	done
	exit 1
}

function build_final_repo() {
	# make sure the repo is clean
	rm -rf $finalrepo/*
	# copy the files, don´t move them for debugging / caching
	find $localmockrepo -type f -name "*.rpm" -exec cp {} $finalrepo \;
	cd $finalrepo
	# create repo skeleton
	mkdir -p rpms/noarch
	mkdir -p rpms/$(uname -m)
	mkdir -p debuginfo
	mkdir -p srpms
	# move stuff in place
	if [ -n "$(ls *.src.rpm 2>/dev/null)" ]; then
		mv *.src.rpm srpms/
	fi
	if [ -n "$(ls *debuginfo*.rpm 2>/dev/null)" ]; then
		mv *debuginfo*.rpm *debugsource*.rpm debuginfo/
	fi
	if [ -n "$(ls *noarch*.rpm 2>/dev/null)" ]; then
		mv *noarch*.rpm rpms/noarch/
	fi
	if [ -n "$(ls *.rpm 2>/dev/null)" ]; then
		mv *.rpm rpms/$(uname -m)/
	fi
	# create repo here
	for i in rpms srpms debuginfo; do
		cd $i
		createrepo .
		cd -
	done
}

function validate_final_repo() {
	case $distro in
		almalinux-9|rhel-9)
			harepo="highavailability"
		;;
		centos-8-stream|rhel-8)
			harepo="ha"
		;;
	esac

	for i in anvil-core anvil-striker anvil-node anvil-dr; do
		mock --enable-network --enablerepo $harepo -r $mocktestconfig --clean
		mock --enable-network --enablerepo $harepo -r $mocktestconfig --install $i
	done
}

function publish_repo() {
	if [ -z "$publish" ]; then
		return
	fi

	installrepo=1
	buildid=$(date +%Y%m%d%H%M%S)

	if [ -d "$publish/$distro/latest" ]; then
		echo "Comparing current build $buildid with latest repos $(cat $publish/$distro/latest/.buildid)"
		latestsrpms="$(ls -1 $publish/$distro/latest/srpms/*.rpm | sed -e 's#.*srpms/##g' | sort -u)"
		currentsrpms="$(ls -1 $finalrepo/srpms/*.rpm | sed -e 's#.*srpms/##g' | sort -u)"
		if [ "$currentsrpms" = "$latestsrpms" ]; then
			echo "Archives contains same rpms, not installing new one"
			installrepo=0
		else
			echo "New archive contains different rpms, installing"
		fi
	fi

	if [ "$installrepo" = "1" ]; then
		mkdir -p $publish/$distro
		mv $finalrepo $publish/$distro/$buildid
		echo $buildid > $publish/$distro/$buildid/.buildid
		cd $publish/$distro
		rm -f latest
		ln -sf $buildid latest
	fi

	echo "Cleaning older archives"
	cd $publish/$distro
	builds="$(ls -1 | grep -v latest | sort -n)"
	numbuilds="$(echo "$builds" | wc -l)"
	if [ "$numbuilds" -gt 5 ]; then
		purgenum=$((numbuilds - 5))
		candidates="$(echo "$builds" | head -n $purgenum)"
		for x in $candidates; do
			echo "Removing old build: $x"
			rm -rf $x
		done
	fi

	echo "Generating repo files"
	cd $publish
	cat > ${distro}-anvil-external-deps.repo << EOF
[${distro}-anvil-external-deps]
name=$distro Anvil external depedencies repo
baseurl=http://anvil-ci-repo.ci.alteeve.com/${distro}/latest/rpms
repo_gpgcheck=0
enabled=1
gpgcheck=0
metadata_expire=1h
skip_if_unavailable=True

[${distro}-anvil-external-deps-debuginfo]
name=$distro Anvil external depedencies repo - Debug
baseurl=http://anvil-ci-repo.ci.alteeve.com/${distro}/latest/debuginfo
repo_gpgcheck=0
enabled=0
gpgcheck=0
metadata_expire=1h
skip_if_unavailable=True

[${distro}-anvil-external-deps-sources]
name=$distro Anvil external depedencies repo - Sources
baseurl=http://anvil-ci-repo.ci.alteeve.com/${distro}/latest/srpms
repo_gpgcheck=0
enabled=0
gpgcheck=0
metadata_expire=1h
skip_if_unavailable=True
EOF

	if selinuxenabled; then
		restorecon -R $publish/
	fi
}

# for now, only simulate drbd-kmod and perl-Log-Journald update
function prep_test_update() {
	rm -rf $tmpsrpms/*
	mkdir -p $tmpsrpms
	for i in drbd-kmod perl-Log-Journald; do
		overridedir=""
		if [ -d "$anvilsrpms/${distro}/$i" ]; then
			overridedir="$anvilsrpms/${distro}/$i"
		elif [ -d "$anvilsrpms/$i" ]; then
			overridedir="$anvilsrpms/$i"
		fi
		cp -rp $overridedir/ $tmpsrpms/
		sed -i \
		    -e 's#^Release:.*#Release: 99%{?dist}#g' \
		    -e 's#-1.el#-99.el#g' \
		    $tmpsrpms/${i}/*.spec
		cd $tmpsrpms/${i}/
		./build.sh
		rm -rf $localsrpms/${i}*
		mv *.src.rpm $localsrpms/
	done
	cd $toplevel
}

# main starts here

# install traps to clean up in case of errors
trap "cleanup 1" ABRT
trap "cleanup 1" QUIT
trap "cleanup 1" TERM
trap "cleanup 1" INT
trap "cleanup 1" ERR

check_requirements
check_distro

cleanup
create_dirs

# init mock
prep_mock_configs
init_mock $mockbuildconfig

# get package list (common to multiple functions)

# sourcing the package-list should return a 'packagelist' variable
# containing the list of source packages to build
. ./configs/${distro}-package-list

prep_srpms
build_rpms

build_final_repo

init_mock $mocktestconfig
validate_final_repo

publish_repo

# generate test update repo
cd $toplevel
finalrepo=$(pwd)/build/${distro}/finalrepo-test-update
create_dirs
prep_test_update
build_rpms
build_final_repo
distro=${distro}-test-update
publish_repo

sleep 10
for i in $(mount | grep workspace | awk '{print $3}'); do umount $i || true; done

cleanup 0