make.cross

#!/bin/bash
#
# linux kernel CROSS make wrapper
#
# It will download/unpack the cross tool chain if necessary,
# then invoke make with suitable options.
#
# It detects ARCH in 4 ways.
#
# - make.i386 # make it a symlink to this script
# - make.cross ARCH=i386
# - cd obj-i386; make.cross
# - export ARCH=i386; make.cross
#
# Specify compiler and version (default is gcc):
# - COMPILER=gcc-9.3.0 make.cross ARCH=arm64
# - COMPILER=clang make.cross ARCH=arm64
#
# Copyright (c) 2014, Intel Corporation.
# Author: Fengguang Wu <fengguang.wu@intel.com>
# Credit: Tony Breeds <tony@bakeyournoodle.com> for crosstool

set -e

lkp_src_kbuild=$(dirname "$(realpath "$0")")

source $lkp_src_kbuild/kbuild.sh

check_install_path()
{
	if [[ $COMPILER_INSTALL_PATH ]]; then
		echo "Compiler will be installed in $COMPILER_INSTALL_PATH"
	else
		return 1
	fi

}

install_packages()
{
	local ret=0
	[[ -x /usr/bin/xz ]] || {
		echo Please install: xz-utils or xz
		ret=1
	}

	[[ -x /usr/bin/lftp ]] || {
		echo Please install: lftp
		ret=1
	}

	[[ -x /usr/bin/ldd ]] || {
		echo Please install: libc-bin
		ret=1
	}

	return $ret
}

install_dependence()
{
	local exec=$1
	local errors=$(ldd $exec | grep "not found")
	[[ $errors ]] || return 0

	echo "$errors" | grep -q "libc.so.6.*GLIBC_" && {
		echo Please update: libc6 or glibc
	}

	echo "ldd $exec"
	echo "$errors"
	return 1
}

download_extract()
{
	local URL="$1"
	local file="$(basename $URL)"
	local dir="$COMPILER_INSTALL_PATH/$(basename $(dirname $URL))"

	mkdir -p $COMPILER_INSTALL_PATH || {
		echo "Can't use $COMPILER_INSTALL_PATH as compiler install path, please choose a different COMPILER_INSTALL_PATH"
		return 1
	}

	mkdir -p $dir || return
	cd $dir || return

	echo lftpget -c $URL
	     lftpget -c $URL || return
	cd - || return

	echo tar Jxf $dir/$file -C $COMPILER_INSTALL_PATH
	tar Jxf $dir/$file -C $COMPILER_INSTALL_PATH
}

show_url_hint()
{
	echo "Please set new URL env variable and rerun"
	echo "  * crosstool provided by kernel org:"
	if [[ $COMPILER =~ clang ]]; then
		echo "    export URL=https://cdn.kernel.org/pub/tools/llvm/files"
	else
		echo "    export URL=https://cdn.kernel.org/pub/tools/crosstool/files/bin/x86_64"
	fi
	echo "  * crosstool provided by 0-Day CI:"
	echo "    export URL=https://download.01.org/0day-ci/cross-package"
}

get_toolchain_list()
{
	timeout 3m lftp -c "open $URL/ && find -d 3" | sort -V > $list || {
		echo "Cannot get list from $URL"
		show_url_hint
		return 1
	}

	return 0
}

get_installed_cross_gcc()
{
	local cross_gcc_candidates=($COMPILER_INSTALL_PATH/${COMPILER}*/${crosstool}/bin/${crosstool}-gcc)
	local cross_gcc=$(echo "${cross_gcc_candidates[@]}" | tr ' ' '\n' | sort -V | tail -1)
	[[ -x $cross_gcc ]] && echo $cross_gcc
}

get_installed_cross_clang()
{
	local cross_clang
	if [[ $URL =~ cdn.kernel.org ]]; then
		local cross_clang_candidates=($COMPILER_INSTALL_PATH/${COMPILER//clang/llvm}*/bin/clang)
		cross_clang=$(echo "${cross_clang_candidates[@]}" | tr ' ' '\n' | sort -V | tail -1)
	else
		cross_clang=$COMPILER_INSTALL_PATH/$COMPILER/bin/clang
	fi

	[[ -x $cross_clang ]] && echo $cross_clang
}

install_crosstool_clang()
{
	[[ $URL ]] || URL='https://cdn.kernel.org/pub/tools/llvm/files'

	local cross_clang=$(get_installed_cross_clang)
	[[ $cross_clang ]] && return

	local file
	if [[ $URL =~ cdn.kernel.org ]]; then
		# ./llvm-13.0.1-x86_64.tar.xz
		# ./llvm-14.0.6-x86_64.tar.xz
		# ./llvm-15.0.7-x86_64.tar.xz
		# ./llvm-16.0.0-x86_64.tar.xz
		# ./llvm-16.0.1-x86_64.tar.xz
		# ./llvm-17.0.3-x86_64.tar.xz
		# ./llvm-17.0.4-x86_64.tar.xz
		local list=/tmp/0day-ci-llvm-files
		get_toolchain_list || return

		file=$(grep -E "${COMPILER//clang/llvm}\..*-x86_64\.tar\.xz$" $list | tail -1)
		rm $list
	else
		file="$COMPILER/$COMPILER.tar.xz"
	fi

	[[ $file ]] || {
		echo "Cannot find $COMPILER under $URL"
		show_url_hint
		return 1
	}

	download_extract "$URL/$file" || {
		echo "Failed to download $URL/$file"
		return 1
	}
}

install_crosstool_gcc()
{
	local cross_gcc=$(get_installed_cross_gcc)
	[[ $cross_gcc ]] && return

	[[ $URL ]] || local URL='https://cdn.kernel.org/pub/tools/crosstool/files/bin/x86_64'

	local list=/tmp/0day-ci-crosstool-files
	get_toolchain_list || return

	local file
	local gcc_arch_pattern=$(echo "${crosstool}" | sed 's/*/.*/g')

	local gcc_version
	[[ $COMPILER =~ - ]] && gcc_version=${COMPILER##*-}

	if [[ $gcc_version ]]; then
		# for arch has more than 1 cross tool available, like x86_64-gcc-9.3.0-nolibc_arm-linux-gnueabi.tar.xz
		# and x86_64-gcc-9.3.0-nolibc_arm-linux-gnueabihf.tar.xz for arm, match "arm-linux-gnueabi." instead of
		# "arm-linux-gnueabi" to get single tool package which match exactly.
		file=$(grep "${gcc_arch_pattern}\..*tar\.xz$" $list | grep "${gcc_version}" | tail -1)
	else
		file=$(grep "${gcc_arch_pattern}\..*tar\.xz$" $list | tail -1)
	fi

	[[ $file ]] || {
		echo "Cannot find $gcc_arch_pattern under $URL check $list"
		show_url_hint
		return 1
	}

	rm $list

	download_extract "$URL/$file" || {
		echo "Failed to download $URL/$file"
		return 1
	}
}

install_crosstool()
{
	if [[ "$COMPILER" =~ "clang" ]]; then
		install_crosstool_clang || {
			echo "clang crosstool install failed"
			return 1
		}
	else
		install_crosstool_gcc || {
			echo "gcc crosstool install failed"
			return 1
		}
	fi
}

install_cross_compiler()
{
	install_packages && install_crosstool
}

setup_gcc_exec()
{
	install_cross_compiler || {
		echo "Install gcc cross compiler failed"
		return 1
	}

	gcc_exec=$(get_installed_cross_gcc)
	[[ $gcc_exec ]] || {
		echo "No gcc cross compiler for $ARCH"
		return 1
	}

	# load build-in depends libs
	local deplibs_path=($COMPILER_INSTALL_PATH/${COMPILER}*/${crosstool}/libexec/gcc/${crosstool}/*)
	deplibs_path=$(tr ' ' '\n' <<< ${deplibs_path[@]} | sort -V | tail -n1)
	[[ -d $deplibs_path ]] && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$deplibs_path

	install_dependence "$gcc_exec"
}

update_path_env_for_parisc()
{
	local crosstool=$1
	setup_gcc_exec || return
	gcc_path=$(dirname $gcc_exec)
	[[ $PATH =~ $gcc_path ]] || export PATH=$gcc_path:$PATH
}

setup_parisc_crosstool()
{
	# 64bit build needs both hppa-linux and hppa64-linux
	# > Assembler messages:
	# > ../arch/parisc/kernel/vdso32/sigtramp.S:39: Error: unknown pseudo-op: `.proc'
	# > ../arch/parisc/kernel/vdso32/restart_syscall.S:16: Error: bad or irreducible absolute expression
	# > ../arch/parisc/kernel/vdso32/sigtramp.S:40: Error: unknown pseudo-op: `.callinfo'
	update_path_env_for_parisc hppa-linux || return
	[[ $crosstool =~ hppa64-linux ]] && {
		update_path_env_for_parisc hppa64-linux || return
	}
	return 0

	# https://lore.kernel.org/lkml/570878b9-a9c8-a12e-4721-32b6ca010995@gmx.de/
	# Please drop the "CROSS_COMPILE=hppa64-linux-" part.
	# It will autodetect the cross compiler, so this is sufficient:
	# make W=1 --keep-going -j32 O=./build_dir ARCH=parisc64 prepare
}

setup_crosstool_gcc()
{
	local crosstool
	local gcc_exec

	setup_cross_vars

	if [[ $crosstool ]]; then
		if [[ $crosstool =~ hppa ]]; then
			setup_parisc_crosstool || return
		else
			setup_gcc_exec || return
			opt_cross="CROSS_COMPILE=${gcc_exec%gcc}"
		fi
	else
		opt_cross=
	fi

	setup_kcflags "$gcc_exec"

	echo "PATH=$PATH"
	return 0
}

setup_crosstool_clang()
{
	install_cross_compiler || {
		echo "Install clang compiler failed"
		return 1
	}

	local cross_clang=$(get_installed_cross_clang)
	[[ $cross_clang ]] || return

	install_dependence "$cross_clang" || return

	local cross_clang_path=$(dirname $cross_clang)
	local opt_ldd="LD=${cross_clang_path}/ld.lld HOSTLD=${cross_clang_path}/ld.lld"
	local opt_objcopy="OBJCOPY=llvm-objcopy"
	[[ $ARCH = s390 ]] && {
		opt_ldd=
		# from binutils-s390x-linux-gnu
		opt_objcopy="OBJCOPY=/usr/s390x-linux-gnu/bin/objcopy"
	}

	export PATH="$cross_clang_path:$PATH"
	echo "PATH=$PATH"

	local kernel_version=$(make kernelversion)
	local kernel_version_major=$(echo "$kernel_version" | cut -d. -f1)
	local kernel_version_minor=$(echo "$kernel_version" | cut -d. -f2)

	# https://www.kernel.org/doc/html/latest/kbuild/llvm.html
	if is_llvm_equal_one_supported; then
		opt_cross="LLVM=1"
	else
		opt_cross="HOSTCC=$cross_clang CC=$cross_clang $opt_ldd $opt_objcopy \
			AR=llvm-ar NM=llvm-nm STRIP=llvm-strip OBJDUMP=llvm-objdump OBJSIZE=llvm-size READELF=llvm-readelf HOSTCXX=clang++ HOSTAR=llvm-ar"
	fi

	# TODO: CROSS_COMPILE is not necessary for new kernel
	setup_cross_vars
	[[ $ARCH = s390 ]] && crosstool=s390x-linux
	opt_cross="$opt_cross CROSS_COMPILE=$crosstool-"

	local opt_llvm_ias=$(setup_llvm_ias "$opt_cross")
	[[ $opt_llvm_ias ]] && opt_cross="$opt_cross $opt_llvm_ias"

	setup_kcflags "$cross_clang"
	:
}

setup_crosstool()
{
	if [[ "$COMPILER" =~ "clang" ]]; then
		setup_crosstool_clang
	else
		setup_crosstool_gcc
	fi
}

setup_kcflags()
{
	local compiler_bin=$1
	local kcflags=

	add_kbuild_kcflags $lkp_src_kbuild/etc/kbuild-kcflags

	[[ $kcflags ]] && opt_kbuild_cflags+=(KCFLAGS="$kcflags")
}

shopt -s nullglob

COMPILER=${COMPILER:-"gcc"}

if [[ ! "$0" =~ 'make.cross' && "$0" =~ make\.([a-z0-9_]+) ]]; then
	export ARCH="${BASH_REMATCH[1]}"
elif [[ "$*" =~ ARCH=([a-z0-9_]+) ]]; then
	export ARCH="${BASH_REMATCH[1]}"
elif [[ ${PWD##*-} =~ ^($(echo $KBUILD_SUPPORTED_ARCHS | sed 's/ /|/g'))$ ]]; then
	echo "ARCH set to ${PWD##*-}"
	export ARCH=${PWD##*-}
elif [[ ! $ARCH ]]; then
	export ARCH=x86_64
fi

[[ "$*" =~ ARCH=([a-z0-9_]+) ]] && [[ $ARCH != ${BASH_REMATCH[1]} ]] && {
	echo "Conflicting ARCH specified! $ARCH ${BASH_REMATCH[1]}"
	exit 1
}

if [[ "$*" =~ O=([^ ]+) ]]; then
	BUILD_DIR="${BASH_REMATCH[1]}"
elif [[ -n "$KBUILD_OUTPUT" ]]; then
	BUILD_DIR="$KBUILD_OUTOUT"
elif [[ -d obj-$ARCH ]]; then
	BUILD_DIR=obj-$ARCH
	O=O=obj-$ARCH
else
	BUILD_DIR="."
fi

check_install_path || {
	echo "Please set COMPILER_INSTALL_PATH to specify compiler install path"
	echo "E.g. COMPILER_INSTALL_PATH=\$HOME/0day COMPILER=clang-17 make.cross ARCH=x86_64"
	echo
	exit 1
}

opt_kbuild_cflags=()
setup_crosstool || {
	echo "setup_crosstool failed"
	exit 1
}

[[ "$*" =~ (-j|--jobs) ]] || {
	nr_cpu=$(getconf _NPROCESSORS_CONF)
	opt_jobs="--jobs=$((nr_cpu * 2))"
}

[[ "$*" =~ "ARCH=$ARCH" ]] || {
	opt_arch="ARCH=$ARCH"
}

[[ -f arch/$(src_arch $ARCH)/boot/dts/Makefile ]] && make_dts="CONFIG_OF_ALL_DTBS=y CONFIG_DTC=y"

[[ -f .make-env ]] && source ./.make-env

if [[ -d source && -L source ]]; then
	echo make --keep-going -C source O=$PWD $make_dts $opt_arch $opt_cross $subarch $opt_jobs "${opt_kbuild_cflags[@]}" "$@"
	exec make --keep-going -C source O=$PWD $make_dts $opt_arch $opt_cross $subarch $opt_jobs "${opt_kbuild_cflags[@]}" "$@"
else
	echo make --keep-going $O $make_dts $opt_arch $opt_cross $subarch $opt_jobs "${opt_kbuild_cflags[@]}" "$@"
	exec make --keep-going $O $make_dts $opt_arch $opt_cross $subarch $opt_jobs "${opt_kbuild_cflags[@]}" "$@"
fi