From 55d30fa5d165a8efadb632603a95329e33bcfa2b Mon Sep 17 00:00:00 2001 From: Ben Cressey Date: Mon, 19 Apr 2021 17:45:51 +0000 Subject: [PATCH 1/7] rename kernel package to kernel-5.4 This allows us to add other kernels to the packages tree. Signed-off-by: Ben Cressey --- README.md | 2 +- packages/{kernel => kernel-5.4}/.gitignore | 0 .../0001-lustrefsx-Disable-Werror-stringop-overflow.patch | 0 ...1-Makefile-add-prepare-target-for-external-modules.patch | 0 packages/{kernel => kernel-5.4}/Cargo.toml | 5 ++++- packages/{kernel => kernel-5.4}/build.rs | 0 packages/{kernel => kernel-5.4}/config-bottlerocket | 0 packages/{kernel/kernel.spec => kernel-5.4/kernel-5.4.spec} | 6 +++++- packages/kernel-5.4/latest-srpm-url.sh | 2 ++ packages/{kernel => kernel-5.4}/pkg.rs | 0 packages/kernel/latest-srpm-url.sh | 2 -- packages/release/Cargo.toml | 2 +- packages/release/release.spec | 4 +--- variants/Cargo.lock | 4 ++-- 14 files changed, 16 insertions(+), 11 deletions(-) rename packages/{kernel => kernel-5.4}/.gitignore (100%) rename packages/{kernel => kernel-5.4}/0001-lustrefsx-Disable-Werror-stringop-overflow.patch (100%) rename packages/{kernel => kernel-5.4}/1001-Makefile-add-prepare-target-for-external-modules.patch (100%) rename packages/{kernel => kernel-5.4}/Cargo.toml (88%) rename packages/{kernel => kernel-5.4}/build.rs (100%) rename packages/{kernel => kernel-5.4}/config-bottlerocket (100%) rename packages/{kernel/kernel.spec => kernel-5.4/kernel-5.4.spec} (97%) create mode 100755 packages/kernel-5.4/latest-srpm-url.sh rename packages/{kernel => kernel-5.4}/pkg.rs (100%) delete mode 100755 packages/kernel/latest-srpm-url.sh diff --git a/README.md b/README.md index cc53e0520cd..96399dbcc11 100644 --- a/README.md +++ b/README.md @@ -578,7 +578,7 @@ We use RPM package definitions to build and install individual packages into an RPM itself is not in the image - it's just a common and convenient package definition format. We currently package the following major third-party components: -* Linux kernel ([background](https://en.wikipedia.org/wiki/Linux), [packaging](packages/kernel/)) +* Linux kernel ([background](https://en.wikipedia.org/wiki/Linux), [packaging](packages/kernel-5.4/)) * glibc ([background](https://www.gnu.org/software/libc/), [packaging](packages/glibc/)) * Buildroot as build toolchain ([background](https://buildroot.org/), via the [SDK](https://github.com/bottlerocket-os/bottlerocket-sdk)) * GRUB, with patches for partition flip updates ([background](https://www.gnu.org/software/grub/), [packaging](packages/grub/)) diff --git a/packages/kernel/.gitignore b/packages/kernel-5.4/.gitignore similarity index 100% rename from packages/kernel/.gitignore rename to packages/kernel-5.4/.gitignore diff --git a/packages/kernel/0001-lustrefsx-Disable-Werror-stringop-overflow.patch b/packages/kernel-5.4/0001-lustrefsx-Disable-Werror-stringop-overflow.patch similarity index 100% rename from packages/kernel/0001-lustrefsx-Disable-Werror-stringop-overflow.patch rename to packages/kernel-5.4/0001-lustrefsx-Disable-Werror-stringop-overflow.patch diff --git a/packages/kernel/1001-Makefile-add-prepare-target-for-external-modules.patch b/packages/kernel-5.4/1001-Makefile-add-prepare-target-for-external-modules.patch similarity index 100% rename from packages/kernel/1001-Makefile-add-prepare-target-for-external-modules.patch rename to packages/kernel-5.4/1001-Makefile-add-prepare-target-for-external-modules.patch diff --git a/packages/kernel/Cargo.toml b/packages/kernel-5.4/Cargo.toml similarity index 88% rename from packages/kernel/Cargo.toml rename to packages/kernel-5.4/Cargo.toml index 749ebe8af5b..9f56c579533 100644 --- a/packages/kernel/Cargo.toml +++ b/packages/kernel-5.4/Cargo.toml @@ -1,10 +1,13 @@ [package] -name = "kernel" +name = "kernel-5_4" version = "0.1.0" edition = "2018" publish = false build = "build.rs" +[package.metadata.build-package] +package-name = "kernel-5.4" + [lib] path = "pkg.rs" diff --git a/packages/kernel/build.rs b/packages/kernel-5.4/build.rs similarity index 100% rename from packages/kernel/build.rs rename to packages/kernel-5.4/build.rs diff --git a/packages/kernel/config-bottlerocket b/packages/kernel-5.4/config-bottlerocket similarity index 100% rename from packages/kernel/config-bottlerocket rename to packages/kernel-5.4/config-bottlerocket diff --git a/packages/kernel/kernel.spec b/packages/kernel-5.4/kernel-5.4.spec similarity index 97% rename from packages/kernel/kernel.spec rename to packages/kernel-5.4/kernel-5.4.spec index 617932ba2dc..96c9c5591aa 100644 --- a/packages/kernel/kernel.spec +++ b/packages/kernel-5.4/kernel-5.4.spec @@ -1,6 +1,6 @@ %global debug_package %{nil} -Name: %{_cross_os}kernel +Name: %{_cross_os}kernel-5.4 Version: 5.4.105 Release: 1%{?dist} Summary: The Linux kernel @@ -22,6 +22,10 @@ BuildRequires: hostname BuildRequires: kmod BuildRequires: openssl-devel +# Pull in expected modules and development files. +Requires: %{name}-modules = %{version}-%{release} +Requires: %{name}-devel = %{version}-%{release} + %global kernel_sourcedir %{_cross_usrsrc}/kernels %global kernel_libdir %{_cross_libdir}/modules/%{version} diff --git a/packages/kernel-5.4/latest-srpm-url.sh b/packages/kernel-5.4/latest-srpm-url.sh new file mode 100755 index 00000000000..5e9e4591d00 --- /dev/null +++ b/packages/kernel-5.4/latest-srpm-url.sh @@ -0,0 +1,2 @@ +#!/bin/sh +docker run --rm amazonlinux:2 sh -c 'amazon-linux-extras enable kernel-5.4 >/dev/null && yum install -q -y yum-utils && yumdownloader -q --source --urls kernel | grep ^http' diff --git a/packages/kernel/pkg.rs b/packages/kernel-5.4/pkg.rs similarity index 100% rename from packages/kernel/pkg.rs rename to packages/kernel-5.4/pkg.rs diff --git a/packages/kernel/latest-srpm-url.sh b/packages/kernel/latest-srpm-url.sh deleted file mode 100755 index 3913cef9cac..00000000000 --- a/packages/kernel/latest-srpm-url.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -docker run --rm amazonlinux:2 sh -c 'amazon-linux-extras enable kernel-ng >/dev/null && yum install -q -y yum-utils && yumdownloader -q --source --urls kernel | grep ^http' diff --git a/packages/release/Cargo.toml b/packages/release/Cargo.toml index 86dd0a284f3..a5e59a56e90 100644 --- a/packages/release/Cargo.toml +++ b/packages/release/Cargo.toml @@ -24,7 +24,7 @@ filesystem = { path = "../filesystem" } glibc = { path = "../glibc" } grub = { path = "../grub" } iproute = { path = "../iproute" } -kernel = { path = "../kernel" } +kernel-5_4 = { path = "../kernel-5.4" } libaudit = { path = "../libaudit" } libgcc = { path = "../libgcc" } libstd-rust = { path = "../libstd-rust" } diff --git a/packages/release/release.spec b/packages/release/release.spec index 7f5812c228b..f71e9c8dfcd 100644 --- a/packages/release/release.spec +++ b/packages/release/release.spec @@ -52,9 +52,7 @@ Requires: %{_cross_os}filesystem Requires: %{_cross_os}glibc Requires: %{_cross_os}grub Requires: %{_cross_os}iproute -Requires: %{_cross_os}kernel -Requires: %{_cross_os}kernel-modules -Requires: %{_cross_os}kernel-devel +Requires: %{_cross_os}kernel-5.4 Requires: %{_cross_os}selinux-policy Requires: %{_cross_os}policycoreutils Requires: %{_cross_os}systemd diff --git a/variants/Cargo.lock b/variants/Cargo.lock index bc2970bd911..e35299138d4 100644 --- a/variants/Cargo.lock +++ b/variants/Cargo.lock @@ -296,7 +296,7 @@ dependencies = [ ] [[package]] -name = "kernel" +name = "kernel-5_4" version = "0.1.0" dependencies = [ "filesystem", @@ -610,7 +610,7 @@ dependencies = [ "glibc", "grub", "iproute", - "kernel", + "kernel-5_4", "libaudit", "libgcc", "libstd-rust", From e218e79de6da2a3dc6218c0fbb36868160d83fd0 Mon Sep 17 00:00:00 2001 From: Ben Cressey Date: Mon, 19 Apr 2021 19:06:30 +0000 Subject: [PATCH 2/7] add kernel 5.10 package Signed-off-by: Ben Cressey --- packages/kernel-5.10/.gitignore | 1 + ...-prepare-target-for-external-modules.patch | 50 ++++ packages/kernel-5.10/Cargo.toml | 17 ++ packages/kernel-5.10/build.rs | 9 + packages/kernel-5.10/config-bottlerocket | 63 +++++ packages/kernel-5.10/kernel-5.10.spec | 239 ++++++++++++++++++ packages/kernel-5.10/latest-srpm-url.sh | 2 + packages/kernel-5.10/pkg.rs | 1 + 8 files changed, 382 insertions(+) create mode 100644 packages/kernel-5.10/.gitignore create mode 100644 packages/kernel-5.10/1001-Makefile-add-prepare-target-for-external-modules.patch create mode 100644 packages/kernel-5.10/Cargo.toml create mode 100644 packages/kernel-5.10/build.rs create mode 100644 packages/kernel-5.10/config-bottlerocket create mode 100644 packages/kernel-5.10/kernel-5.10.spec create mode 100755 packages/kernel-5.10/latest-srpm-url.sh create mode 100644 packages/kernel-5.10/pkg.rs diff --git a/packages/kernel-5.10/.gitignore b/packages/kernel-5.10/.gitignore new file mode 100644 index 00000000000..f0af3ba1722 --- /dev/null +++ b/packages/kernel-5.10/.gitignore @@ -0,0 +1 @@ +kernel-*.src.rpm diff --git a/packages/kernel-5.10/1001-Makefile-add-prepare-target-for-external-modules.patch b/packages/kernel-5.10/1001-Makefile-add-prepare-target-for-external-modules.patch new file mode 100644 index 00000000000..13da689ce62 --- /dev/null +++ b/packages/kernel-5.10/1001-Makefile-add-prepare-target-for-external-modules.patch @@ -0,0 +1,50 @@ +From b6d859b7089dd68d3186f2a088823c322ad4852e Mon Sep 17 00:00:00 2001 +From: Ben Cressey +Date: Mon, 19 Apr 2021 18:46:04 +0000 +Subject: [PATCH] Makefile: add prepare target for external modules + +We need to ensure that native versions of programs like `objtool` are +built before trying to build out-of-tree modules, or else the build +will fail. + +Unlike other distributions, we cannot include these programs in our +kernel-devel archive, because we rely on cross-compilation: these are +"host" programs and may not match the architecture of the target. + +Ideally, out-of-tree builds would run `make prepare` first, so that +these programs could be compiled in the normal fashion. We ship all +the files needed for this to work. However, this requirement is +specific to our use case, and DKMS does not support it. + +Adding a minimal prepare target to the dependency graph causes the +programs to be built automatically and improves compatibility with +existing solutions. + +Signed-off-by: Ben Cressey +--- + Makefile | 9 +++++++++ + 1 file changed, 9 insertions(+) + +diff --git a/Makefile b/Makefile +index 1d4a50ebe3b7..b9347d1e69e2 100644 +--- a/Makefile ++++ b/Makefile +@@ -1719,6 +1719,15 @@ else # KBUILD_EXTMOD + KBUILD_BUILTIN := + KBUILD_MODULES := 1 + ++PHONY += modules_prepare ++modules_prepare: $(objtool_target) ++ $(Q)$(MAKE) $(build)=scripts/basic ++ $(Q)$(MAKE) $(build)=scripts/dtc ++ $(Q)$(MAKE) $(build)=scripts/mod ++ $(Q)$(MAKE) $(build)=scripts ++ ++prepare: modules_prepare ++ + build-dirs := $(KBUILD_EXTMOD) + PHONY += modules + modules: $(MODORDER) +-- +2.21.3 + diff --git a/packages/kernel-5.10/Cargo.toml b/packages/kernel-5.10/Cargo.toml new file mode 100644 index 00000000000..bd608ef0db5 --- /dev/null +++ b/packages/kernel-5.10/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "kernel-5_10" +version = "0.1.0" +edition = "2018" +publish = false +build = "build.rs" + +[package.metadata.build-package] +package-name = "kernel-5.10" + +[lib] +path = "pkg.rs" + +[[package.metadata.build-package.external-files]] +# Use latest-srpm-url.sh to get this. +url = "https://cdn.amazonlinux.com/blobstore/fa04b98fc067a4943beac60d0c2971e2fbef1a29faed4bac1c4096abe4ad4c12/kernel-5.10.29-27.126.amzn2.src.rpm" +sha512 = "47341f4a1c13ba7e5ea72bad13fe689eefd22cc7547aea08a08fe47238b4a3fe1659786a406b84a1d1508143be20d9be2fae6fe3e7a6924bc85043bf61d4bfce" diff --git a/packages/kernel-5.10/build.rs b/packages/kernel-5.10/build.rs new file mode 100644 index 00000000000..cad8999af53 --- /dev/null +++ b/packages/kernel-5.10/build.rs @@ -0,0 +1,9 @@ +use std::process::{exit, Command}; + +fn main() -> Result<(), std::io::Error> { + let ret = Command::new("buildsys").arg("build-package").status()?; + if !ret.success() { + exit(1); + } + Ok(()) +} diff --git a/packages/kernel-5.10/config-bottlerocket b/packages/kernel-5.10/config-bottlerocket new file mode 100644 index 00000000000..26cce3d6486 --- /dev/null +++ b/packages/kernel-5.10/config-bottlerocket @@ -0,0 +1,63 @@ +# Because Bottlerocket does not have an initramfs, modules required to mount +# the root filesystem must be set to y. + +# The root filesystem is ext4 +CONFIG_EXT4_FS=y + +# NVMe for EC2 Nitro platforms (C5, M5, and later) +CONFIG_BLK_DEV_NVME=y +CONFIG_NVME_CORE=y + +# Xen blkfront for Xen-based EC2 platforms +CONFIG_XEN_BLKDEV_FRONTEND=y + +# virtio for local testing with QEMU +CONFIG_VIRTIO=y +CONFIG_VIRTIO_BLK=y +CONFIG_VIRTIO_PCI=y + +# dm-verity and enabling it on the kernel command line +CONFIG_BLK_DEV_DM=y +CONFIG_DAX=y +CONFIG_DM_INIT=y +CONFIG_DM_VERITY=y + +# yama LSM for ptrace restrictions +CONFIG_SECURITY_YAMA=y + +# Do not allow SELinux to be disabled at boot. +CONFIG_SECURITY_SELINUX_BOOTPARAM=n + +# Do not allow SELinux to be disabled at runtime. +CONFIG_SECURITY_SELINUX_DISABLE=n + +# Do not allow SELinux to use `enforcing=0` behavior. +CONFIG_SECURITY_SELINUX_DEVELOP=n + +# Check the protection applied by the kernel for mmap and mprotect, +# rather than the protection requested by userspace. +CONFIG_SECURITY_SELINUX_CHECKREQPROT_VALUE=0 + +# Enable support for the kernel lockdown security module. +CONFIG_SECURITY_LOCKDOWN_LSM=y + +# Enable lockdown early so that if the option is present on the +# kernel command line, it can be enforced. +CONFIG_SECURITY_LOCKDOWN_LSM_EARLY=y + +# Enable zstd compression for squashfs. +CONFIG_SQUASHFS_ZSTD=y + +# enable /proc/config.gz +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y + +# kernel headers at /sys/kernel/kheaders.tar.xz +CONFIG_IKHEADERS=y + +# BTF debug info at /sys/kernel/btf/vmlinux +CONFIG_DEBUG_INFO_BTF=y + +# We don't want to extend the kernel command line with any upstream defaults; +# Bottlerocket uses a fairly custom setup that needs tight control over it. +CONFIG_CMDLINE_EXTEND=n diff --git a/packages/kernel-5.10/kernel-5.10.spec b/packages/kernel-5.10/kernel-5.10.spec new file mode 100644 index 00000000000..5da57565011 --- /dev/null +++ b/packages/kernel-5.10/kernel-5.10.spec @@ -0,0 +1,239 @@ +%global debug_package %{nil} + +Name: %{_cross_os}kernel-5.10 +Version: 5.10.29 +Release: 1%{?dist} +Summary: The Linux kernel +License: GPL-2.0 WITH Linux-syscall-note +URL: https://www.kernel.org/ +# Use latest-srpm-url.sh to get this. +Source0: https://cdn.amazonlinux.com/blobstore/fa04b98fc067a4943beac60d0c2971e2fbef1a29faed4bac1c4096abe4ad4c12/kernel-5.10.29-27.126.amzn2.src.rpm +Source100: config-bottlerocket + +# Help out-of-tree module builds run `make prepare` automatically. +Patch1001: 1001-Makefile-add-prepare-target-for-external-modules.patch + +BuildRequires: bc +BuildRequires: elfutils-devel +BuildRequires: hostname +BuildRequires: kmod +BuildRequires: openssl-devel + +# Pull in expected modules and development files. +Requires: %{name}-modules = %{version}-%{release} +Requires: %{name}-devel = %{version}-%{release} + +%global kernel_sourcedir %{_cross_usrsrc}/kernels +%global kernel_libdir %{_cross_libdir}/modules/%{version} + +%description +%{summary}. + +%package devel +Summary: Configured Linux kernel source for module building +Requires: %{_cross_os}filesystem + +%description devel +%{summary}. + +%package archive +Summary: Archived Linux kernel source for module building + +%description archive +%{summary}. + +%package modules +Summary: Modules for the Linux kernel + +%description modules +%{summary}. + +%package headers +Summary: Header files for the Linux kernel for use by glibc + +%description headers +%{summary}. + +%prep +rpm2cpio %{SOURCE0} | cpio -iu linux-%{version}.tar config-%{_cross_arch} "*.patch" +tar -xof linux-%{version}.tar; rm linux-%{version}.tar +%setup -TDn linux-%{version} +# Patches from the Source0 SRPM +for patch in ../*.patch; do + patch -p1 <"$patch" +done +# Patches listed in this spec (Patch0001...) +%autopatch -p1 +KCONFIG_CONFIG="arch/%{_cross_karch}/configs/%{_cross_vendor}_defconfig" \ + ARCH="%{_cross_karch}" \ + scripts/kconfig/merge_config.sh ../config-%{_cross_arch} %{SOURCE100} +rm -f ../config-%{_cross_arch} ../*.patch + +%global kmake \ +make -s\\\ + ARCH="%{_cross_karch}"\\\ + CROSS_COMPILE="%{_cross_target}-"\\\ + INSTALL_HDR_PATH="%{buildroot}%{_cross_prefix}"\\\ + INSTALL_MOD_PATH="%{buildroot}%{_cross_prefix}"\\\ + INSTALL_MOD_STRIP=1\\\ +%{nil} + +%build +%kmake mrproper +%kmake %{_cross_vendor}_defconfig +%kmake %{?_smp_mflags} %{_cross_kimage} +%kmake %{?_smp_mflags} modules + +%install +%kmake headers_install +%kmake modules_install + +install -d %{buildroot}/boot +install -T -m 0755 arch/%{_cross_karch}/boot/%{_cross_kimage} %{buildroot}/boot/vmlinuz +install -m 0644 .config %{buildroot}/boot/config +install -m 0644 System.map %{buildroot}/boot/System.map + +find %{buildroot}%{_cross_prefix} \ + \( -name .install -o -name .check -o \ + -name ..install.cmd -o -name ..check.cmd \) -delete + +# For out-of-tree kmod builds, we need to support the following targets: +# make scripts -> make prepare -> make modules +# +# This requires enough of the kernel tree to build host programs under the +# "scripts" and "tools" directories. + +# Any existing ELF objects will not work properly if we're cross-compiling for +# a different architecture, so get rid of them to avoid confusing errors. +find arch scripts tools -type f -executable \ + -exec sh -c "head -c4 {} | grep -q ELF && rm {}" \; + +# We don't need to include these files. +find -type f \( -name \*.cmd -o -name \*.gitignore \) -delete + +# Avoid an OpenSSL dependency by stubbing out options for module signing and +# trusted keyrings, so `sign-file` and `extract-cert` won't be built. External +# kernel modules do not have access to the keys they would need to make use of +# these tools. +sed -i \ + -e 's,$(CONFIG_MODULE_SIG_FORMAT),n,g' \ + -e 's,$(CONFIG_SYSTEM_TRUSTED_KEYRING),n,g' \ + scripts/Makefile + +( + find * \ + -type f \ + \( -name Build\* -o -name Kbuild\* -o -name Kconfig\* -o -name Makefile\* \) \ + -print + + find arch/%{_cross_karch}/ \ + -type f \ + \( -name module.lds -o -name vmlinux.lds.S -o -name Platform -o -name \*.tbl \) \ + -print + + find arch/%{_cross_karch}/{include,lib}/ -type f ! -name \*.o ! -name \*.o.d -print + echo arch/%{_cross_karch}/kernel/asm-offsets.s + echo lib/vdso/gettimeofday.c + + for d in \ + arch/%{_cross_karch}/tools \ + arch/%{_cross_karch}/kernel/vdso ; do + [ -d "${d}" ] && find "${d}/" -type f -print + done + + find include -type f -print + find scripts -type f ! -name \*.l ! -name \*.y ! -name \*.o -print + + find tools/{arch/%{_cross_karch},include,objtool,scripts}/ -type f ! -name \*.o -print + echo tools/build/fixdep.c + find tools/lib/subcmd -type f -print + find tools/lib/{ctype,rbtree,string,str_error_r}.c + + echo kernel/bounds.c + echo kernel/time/timeconst.bc + echo security/selinux/include/classmap.h + echo security/selinux/include/initial_sid_to_string.h + echo security/selinux/include/policycap.h + echo security/selinux/include/policycap_names.h + + echo .config + echo Module.symvers + echo System.map +) | sort -u > kernel_devel_files + +# Create squashfs of kernel-devel files (ie. /usr/src/kernels/). +# +# -no-exports: +# The filesystem does not need to be exported via NFS. +# +# -all-root: +# Make all files owned by root rather than the build user. +# +# -comp zstd: +# zstd offers compression ratios like xz and decompression speeds like lz4. +SQUASHFS_OPTS="-no-exports -all-root -comp zstd" +mkdir -p src_squashfs/%{version} +tar c -T kernel_devel_files | tar x -C src_squashfs/%{version} +mksquashfs src_squashfs kernel-devel.squashfs ${SQUASHFS_OPTS} + +# Create a tarball of the same files, for use outside the running system. +# In theory we could extract these files with `unsquashfs`, but we do not want +# to require it to be installed on the build host, and it errors out when run +# inside Docker unless the limit for open files is lowered. +tar cf kernel-devel.tar src_squashfs/%{version} --transform='s|src_squashfs/%{version}|kernel-devel|' +xz -T0 kernel-devel.tar + +install -D kernel-devel.squashfs %{buildroot}%{_cross_datadir}/bottlerocket/kernel-devel.squashfs +install -D kernel-devel.tar.xz %{buildroot}%{_cross_datadir}/bottlerocket/kernel-devel.tar.xz +install -d %{buildroot}%{kernel_sourcedir} + +# Replace the incorrect links from modules_install. These will be bound +# into a host container (and unused in the host) so they must not point +# to %{_cross_usrsrc} (eg. /x86_64-bottlerocket-linux-gnu/sys-root/...) +rm -f %{buildroot}%{kernel_libdir}/build %{buildroot}%{kernel_libdir}/source +ln -sf %{_usrsrc}/kernels/%{version} %{buildroot}%{kernel_libdir}/build +ln -sf %{_usrsrc}/kernels/%{version} %{buildroot}%{kernel_libdir}/source + +%files +%license COPYING LICENSES/preferred/GPL-2.0 LICENSES/exceptions/Linux-syscall-note +%{_cross_attribution_file} +/boot/vmlinuz +/boot/config +/boot/System.map + +%files modules +%dir %{_cross_libdir}/modules +%{_cross_libdir}/modules/* + +%files headers +%dir %{_cross_includedir}/asm +%dir %{_cross_includedir}/asm-generic +%dir %{_cross_includedir}/drm +%dir %{_cross_includedir}/linux +%dir %{_cross_includedir}/misc +%dir %{_cross_includedir}/mtd +%dir %{_cross_includedir}/rdma +%dir %{_cross_includedir}/scsi +%dir %{_cross_includedir}/sound +%dir %{_cross_includedir}/video +%dir %{_cross_includedir}/xen +%{_cross_includedir}/asm/* +%{_cross_includedir}/asm-generic/* +%{_cross_includedir}/drm/* +%{_cross_includedir}/linux/* +%{_cross_includedir}/misc/* +%{_cross_includedir}/mtd/* +%{_cross_includedir}/rdma/* +%{_cross_includedir}/scsi/* +%{_cross_includedir}/sound/* +%{_cross_includedir}/video/* +%{_cross_includedir}/xen/* + +%files devel +%dir %{kernel_sourcedir} +%{_cross_datadir}/bottlerocket/kernel-devel.squashfs + +%files archive +%{_cross_datadir}/bottlerocket/kernel-devel.tar.xz + +%changelog diff --git a/packages/kernel-5.10/latest-srpm-url.sh b/packages/kernel-5.10/latest-srpm-url.sh new file mode 100755 index 00000000000..46001c3385f --- /dev/null +++ b/packages/kernel-5.10/latest-srpm-url.sh @@ -0,0 +1,2 @@ +#!/bin/sh +docker run --rm amazonlinux:2 sh -c 'amazon-linux-extras enable kernel-5.10 >/dev/null && yum install -q -y yum-utils && yumdownloader -q --source --urls kernel | grep ^http' diff --git a/packages/kernel-5.10/pkg.rs b/packages/kernel-5.10/pkg.rs new file mode 100644 index 00000000000..d799fb2d44c --- /dev/null +++ b/packages/kernel-5.10/pkg.rs @@ -0,0 +1 @@ +// not used From 55f3ad48fd64231532dadf4e98e41eb644d45ab9 Mon Sep 17 00:00:00 2001 From: Ben Cressey Date: Mon, 19 Apr 2021 19:08:27 +0000 Subject: [PATCH 3/7] move kernel dependency to variant definitions Since we have multiple kernel versions available, variants must now define the version they want to include. Signed-off-by: Ben Cressey --- packages/release/Cargo.toml | 1 - packages/release/release.spec | 1 - variants/Cargo.lock | 13 ++++++++++++- variants/aws-dev/Cargo.toml | 2 ++ variants/aws-ecs-1/Cargo.toml | 2 ++ variants/aws-k8s-1.16/Cargo.toml | 12 +++++++----- variants/aws-k8s-1.17/Cargo.toml | 12 +++++++----- variants/aws-k8s-1.18/Cargo.toml | 12 +++++++----- variants/aws-k8s-1.19/Cargo.toml | 12 +++++++----- variants/aws-k8s-1.20/Cargo.toml | 12 +++++++----- variants/vmware-dev/Cargo.toml | 2 ++ 11 files changed, 53 insertions(+), 28 deletions(-) diff --git a/packages/release/Cargo.toml b/packages/release/Cargo.toml index a5e59a56e90..33733d5101f 100644 --- a/packages/release/Cargo.toml +++ b/packages/release/Cargo.toml @@ -24,7 +24,6 @@ filesystem = { path = "../filesystem" } glibc = { path = "../glibc" } grub = { path = "../grub" } iproute = { path = "../iproute" } -kernel-5_4 = { path = "../kernel-5.4" } libaudit = { path = "../libaudit" } libgcc = { path = "../libgcc" } libstd-rust = { path = "../libstd-rust" } diff --git a/packages/release/release.spec b/packages/release/release.spec index f71e9c8dfcd..df49da30a49 100644 --- a/packages/release/release.spec +++ b/packages/release/release.spec @@ -52,7 +52,6 @@ Requires: %{_cross_os}filesystem Requires: %{_cross_os}glibc Requires: %{_cross_os}grub Requires: %{_cross_os}iproute -Requires: %{_cross_os}kernel-5.4 Requires: %{_cross_os}selinux-policy Requires: %{_cross_os}policycoreutils Requires: %{_cross_os}systemd diff --git a/variants/Cargo.lock b/variants/Cargo.lock index e35299138d4..20ed8c49cc0 100644 --- a/variants/Cargo.lock +++ b/variants/Cargo.lock @@ -18,6 +18,7 @@ dependencies = [ "docker-proxy", "findutils", "iputils", + "kernel-5_10", "login", "procps", "release", @@ -34,6 +35,7 @@ dependencies = [ "docker-init", "docker-proxy", "ecs-agent", + "kernel-5_4", "release", ] @@ -51,6 +53,7 @@ dependencies = [ "aws-iam-authenticator", "cni", "cni-plugins", + "kernel-5_4", "kubernetes-1_16", "release", ] @@ -62,6 +65,7 @@ dependencies = [ "aws-iam-authenticator", "cni", "cni-plugins", + "kernel-5_4", "kubernetes-1_17", "release", ] @@ -73,6 +77,7 @@ dependencies = [ "aws-iam-authenticator", "cni", "cni-plugins", + "kernel-5_4", "kubernetes-1_18", "release", ] @@ -84,6 +89,7 @@ dependencies = [ "aws-iam-authenticator", "cni", "cni-plugins", + "kernel-5_4", "kubernetes-1_19", "release", ] @@ -95,6 +101,7 @@ dependencies = [ "aws-iam-authenticator", "cni", "cni-plugins", + "kernel-5_10", "kubernetes-1_20", "release", ] @@ -295,6 +302,10 @@ dependencies = [ "libcap", ] +[[package]] +name = "kernel-5_10" +version = "0.1.0" + [[package]] name = "kernel-5_4" version = "0.1.0" @@ -610,7 +621,6 @@ dependencies = [ "glibc", "grub", "iproute", - "kernel-5_4", "libaudit", "libgcc", "libstd-rust", @@ -686,6 +696,7 @@ dependencies = [ "docker-proxy", "findutils", "iputils", + "kernel-5_10", "login", "procps", "release", diff --git a/variants/aws-dev/Cargo.toml b/variants/aws-dev/Cargo.toml index 0854c956dc7..dd2eabcf867 100644 --- a/variants/aws-dev/Cargo.toml +++ b/variants/aws-dev/Cargo.toml @@ -15,6 +15,7 @@ kernel-parameters = [ included-packages = [ # core "release", + "kernel-5.10", # docker "docker-cli", "docker-engine", @@ -36,6 +37,7 @@ path = "lib.rs" [build-dependencies] # core release = { path = "../../packages/release" } +kernel-5_10 = { path = "../../packages/kernel-5.10" } # docker docker-cli = { path = "../../packages/docker-cli" } docker-engine = { path = "../../packages/docker-engine" } diff --git a/variants/aws-ecs-1/Cargo.toml b/variants/aws-ecs-1/Cargo.toml index b630da1bf4f..14a3dba9760 100644 --- a/variants/aws-ecs-1/Cargo.toml +++ b/variants/aws-ecs-1/Cargo.toml @@ -13,6 +13,7 @@ kernel-parameters = [ included-packages = [ # core "release", + "kernel-5.4", # docker "docker-cli", "docker-engine", @@ -28,6 +29,7 @@ path = "lib.rs" [build-dependencies] # core release = { path = "../../packages/release" } +kernel-5_4 = { path = "../../packages/kernel-5.4" } # docker docker-cli = { path = "../../packages/docker-cli" } docker-engine = { path = "../../packages/docker-engine" } diff --git a/variants/aws-k8s-1.16/Cargo.toml b/variants/aws-k8s-1.16/Cargo.toml index 02e49f41a4e..5898b307901 100644 --- a/variants/aws-k8s-1.16/Cargo.toml +++ b/variants/aws-k8s-1.16/Cargo.toml @@ -18,6 +18,7 @@ included-packages = [ "aws-iam-authenticator", "cni", "cni-plugins", + "kernel-5.4", "kubelet-1.16", "release", ] @@ -26,8 +27,9 @@ included-packages = [ path = "lib.rs" [build-dependencies] -"aws-iam-authenticator" = { path = "../../packages/aws-iam-authenticator" } -"cni" = { path = "../../packages/cni" } -"cni-plugins" = { path = "../../packages/cni-plugins" } -"kubernetes-1_16" = { path = "../../packages/kubernetes-1.16" } -"release" = { path = "../../packages/release" } +aws-iam-authenticator = { path = "../../packages/aws-iam-authenticator" } +cni = { path = "../../packages/cni" } +cni-plugins = { path = "../../packages/cni-plugins" } +kernel-5_4 = { path = "../../packages/kernel-5.4" } +kubernetes-1_16 = { path = "../../packages/kubernetes-1.16" } +release = { path = "../../packages/release" } diff --git a/variants/aws-k8s-1.17/Cargo.toml b/variants/aws-k8s-1.17/Cargo.toml index 567181e6506..e39aecf5bc0 100644 --- a/variants/aws-k8s-1.17/Cargo.toml +++ b/variants/aws-k8s-1.17/Cargo.toml @@ -18,6 +18,7 @@ included-packages = [ "aws-iam-authenticator", "cni", "cni-plugins", + "kernel-5.4", "kubelet-1.17", "release", ] @@ -26,8 +27,9 @@ included-packages = [ path = "lib.rs" [build-dependencies] -"aws-iam-authenticator" = { path = "../../packages/aws-iam-authenticator" } -"cni" = { path = "../../packages/cni" } -"cni-plugins" = { path = "../../packages/cni-plugins" } -"kubernetes-1_17" = { path = "../../packages/kubernetes-1.17" } -"release" = { path = "../../packages/release" } +aws-iam-authenticator = { path = "../../packages/aws-iam-authenticator" } +cni = { path = "../../packages/cni" } +cni-plugins = { path = "../../packages/cni-plugins" } +kernel-5_4 = { path = "../../packages/kernel-5.4" } +kubernetes-1_17 = { path = "../../packages/kubernetes-1.17" } +release = { path = "../../packages/release" } diff --git a/variants/aws-k8s-1.18/Cargo.toml b/variants/aws-k8s-1.18/Cargo.toml index 72431442030..33a902b1772 100644 --- a/variants/aws-k8s-1.18/Cargo.toml +++ b/variants/aws-k8s-1.18/Cargo.toml @@ -18,6 +18,7 @@ included-packages = [ "aws-iam-authenticator", "cni", "cni-plugins", + "kernel-5.4", "kubelet-1.18", "release", ] @@ -26,8 +27,9 @@ included-packages = [ path = "lib.rs" [build-dependencies] -"aws-iam-authenticator" = { path = "../../packages/aws-iam-authenticator" } -"cni" = { path = "../../packages/cni" } -"cni-plugins" = { path = "../../packages/cni-plugins" } -"kubernetes-1_18" = { path = "../../packages/kubernetes-1.18" } -"release" = { path = "../../packages/release" } +aws-iam-authenticator = { path = "../../packages/aws-iam-authenticator" } +cni = { path = "../../packages/cni" } +cni-plugins = { path = "../../packages/cni-plugins" } +kernel-5_4 = { path = "../../packages/kernel-5.4" } +kubernetes-1_18 = { path = "../../packages/kubernetes-1.18" } +release = { path = "../../packages/release" } diff --git a/variants/aws-k8s-1.19/Cargo.toml b/variants/aws-k8s-1.19/Cargo.toml index e1575413363..860bcdebde9 100644 --- a/variants/aws-k8s-1.19/Cargo.toml +++ b/variants/aws-k8s-1.19/Cargo.toml @@ -18,6 +18,7 @@ included-packages = [ "aws-iam-authenticator", "cni", "cni-plugins", + "kernel-5.4", "kubelet-1.19", "release", ] @@ -26,8 +27,9 @@ included-packages = [ path = "lib.rs" [build-dependencies] -"aws-iam-authenticator" = { path = "../../packages/aws-iam-authenticator" } -"cni" = { path = "../../packages/cni" } -"cni-plugins" = { path = "../../packages/cni-plugins" } -"kubernetes-1_19" = { path = "../../packages/kubernetes-1.19" } -"release" = { path = "../../packages/release" } +aws-iam-authenticator = { path = "../../packages/aws-iam-authenticator" } +cni = { path = "../../packages/cni" } +cni-plugins = { path = "../../packages/cni-plugins" } +kernel-5_4 = { path = "../../packages/kernel-5.4" } +kubernetes-1_19 = { path = "../../packages/kubernetes-1.19" } +release = { path = "../../packages/release" } diff --git a/variants/aws-k8s-1.20/Cargo.toml b/variants/aws-k8s-1.20/Cargo.toml index c89569d34de..b5c5368c611 100644 --- a/variants/aws-k8s-1.20/Cargo.toml +++ b/variants/aws-k8s-1.20/Cargo.toml @@ -14,6 +14,7 @@ included-packages = [ "aws-iam-authenticator", "cni", "cni-plugins", + "kernel-5.10", "kubelet-1.20", "release", ] @@ -27,8 +28,9 @@ kernel-parameters = [ path = "lib.rs" [build-dependencies] -"aws-iam-authenticator" = { path = "../../packages/aws-iam-authenticator" } -"cni" = { path = "../../packages/cni" } -"cni-plugins" = { path = "../../packages/cni-plugins" } -"kubernetes-1_20" = { path = "../../packages/kubernetes-1.20" } -"release" = { path = "../../packages/release" } +aws-iam-authenticator = { path = "../../packages/aws-iam-authenticator" } +cni = { path = "../../packages/cni" } +cni-plugins = { path = "../../packages/cni-plugins" } +kernel-5_10 = { path = "../../packages/kernel-5.10" } +kubernetes-1_20 = { path = "../../packages/kubernetes-1.20" } +release = { path = "../../packages/release" } diff --git a/variants/vmware-dev/Cargo.toml b/variants/vmware-dev/Cargo.toml index 4f3635c0610..3ad6fb8ae3b 100644 --- a/variants/vmware-dev/Cargo.toml +++ b/variants/vmware-dev/Cargo.toml @@ -17,6 +17,7 @@ kernel-parameters = [ included-packages = [ # core "release", + "kernel-5.10", # docker "docker-cli", "docker-engine", @@ -38,6 +39,7 @@ path = "lib.rs" [build-dependencies] # core release = { path = "../../packages/release" } +kernel-5_10 = { path = "../../packages/kernel-5.10" } # docker docker-cli = { path = "../../packages/docker-cli" } docker-engine = { path = "../../packages/docker-engine" } From 45dfb7553a1e50f9b6d9375d5391ef3f9e6470f7 Mon Sep 17 00:00:00 2001 From: Ben Cressey Date: Wed, 28 Apr 2021 14:27:04 +0000 Subject: [PATCH 4/7] kernel: omit filesystem package dependency The "filesystem" package is an implicit dependency for all packages, and it's pulled in by release. Signed-off-by: Ben Cressey --- packages/kernel-5.10/kernel-5.10.spec | 1 - packages/kernel-5.4/Cargo.toml | 4 ---- packages/kernel-5.4/kernel-5.4.spec | 1 - variants/Cargo.lock | 3 --- 4 files changed, 9 deletions(-) diff --git a/packages/kernel-5.10/kernel-5.10.spec b/packages/kernel-5.10/kernel-5.10.spec index 5da57565011..6908fb327e9 100644 --- a/packages/kernel-5.10/kernel-5.10.spec +++ b/packages/kernel-5.10/kernel-5.10.spec @@ -31,7 +31,6 @@ Requires: %{name}-devel = %{version}-%{release} %package devel Summary: Configured Linux kernel source for module building -Requires: %{_cross_os}filesystem %description devel %{summary}. diff --git a/packages/kernel-5.4/Cargo.toml b/packages/kernel-5.4/Cargo.toml index 9f56c579533..aaf0953104f 100644 --- a/packages/kernel-5.4/Cargo.toml +++ b/packages/kernel-5.4/Cargo.toml @@ -19,7 +19,3 @@ sha512 = "ef506706434bc94df6e845e5262c8d022ebb91ff6bc6a71ac656851c0de66d81392ace # RPM BuildRequires [build-dependencies] # Provided by Bottlerocket SDK - -# RPM Requires -[dependencies] -filesystem = { path = "../filesystem" } diff --git a/packages/kernel-5.4/kernel-5.4.spec b/packages/kernel-5.4/kernel-5.4.spec index 96c9c5591aa..3bf79bef30d 100644 --- a/packages/kernel-5.4/kernel-5.4.spec +++ b/packages/kernel-5.4/kernel-5.4.spec @@ -34,7 +34,6 @@ Requires: %{name}-devel = %{version}-%{release} %package devel Summary: Configured Linux kernel source for module building -Requires: %{_cross_os}filesystem %description devel %{summary}. diff --git a/variants/Cargo.lock b/variants/Cargo.lock index 20ed8c49cc0..a7cc20e4c24 100644 --- a/variants/Cargo.lock +++ b/variants/Cargo.lock @@ -309,9 +309,6 @@ version = "0.1.0" [[package]] name = "kernel-5_4" version = "0.1.0" -dependencies = [ - "filesystem", -] [[package]] name = "kmod" From be005b63c93ce759432eca43716f4e1800b485c7 Mon Sep 17 00:00:00 2001 From: Ben Cressey Date: Mon, 19 Apr 2021 20:54:28 +0000 Subject: [PATCH 5/7] selinux-policy: update for kernel 5.10 Enable the new "genfs_seclabel_symlinks" capability to label symlinks on kernel filesystems in the same way that files and directories are labeled. Add the new "perfmon", "bpf", and "checkpoint_restore" actions to the "capability2" and "cap2_userns" classes. Add the new "perf_event" and "lockdown" classes and the corresponding actions. Add the new permissions into the blanket "systems" permission set, so that we continue to rely on the existing kernel access checks without adding new SELinux restrictions. Signed-off-by: Ben Cressey --- packages/selinux-policy/base.cil | 4 ++++ packages/selinux-policy/class.cil | 7 +++++-- packages/selinux-policy/systems.cil | 2 ++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/selinux-policy/base.cil b/packages/selinux-policy/base.cil index 2cc1489765b..4d25e920d39 100644 --- a/packages/selinux-policy/base.cil +++ b/packages/selinux-policy/base.cil @@ -50,3 +50,7 @@ ; "no new privs" bit enabled, or when the executable is on a "nosuid" ; filesystem. (policycap "nnp_nosuid_transition") + +; Enable policy to label symlinks on kernel file systems using +; "genfscon" statements, as with directories and files. +(policycap "genfs_seclabel_symlinks") diff --git a/packages/selinux-policy/class.cil b/packages/selinux-policy/class.cil index 92e395abc6a..2d57bc21d82 100644 --- a/packages/selinux-policy/class.cil +++ b/packages/selinux-policy/class.cil @@ -27,7 +27,7 @@ (common capability2 ( mac_override mac_admin syslog wake_alarm block_suspend - audit_read)) + audit_read perfmon bpf checkpoint_restore)) (classcommon capability2 capability2) (classcommon cap2_userns capability2) @@ -232,6 +232,8 @@ (class kernel_service (use_as_override create_files_as)) (class binder (impersonate call set_context_mgr transfer)) (class bpf (map_create map_read map_write prog_load prog_run)) +(class perf_event (open cpu kernel tracepoint read write)) +(class lockdown (integrity confidentiality)) ; Match the kernel's class order. (classorder ( @@ -256,7 +258,8 @@ bluetooth_socket iucv_socket rxrpc_socket isdn_socket phonet_socket ieee802154_socket caif_socket alg_socket nfc_socket vsock_socket kcm_socket qipcrtr_socket smc_socket - infiniband_pkey infiniband_endport bpf xdp_socket)) + infiniband_pkey infiniband_endport bpf xdp_socket + perf_event lockdown)) ; Add permissions for SELinux-aware applications. ; This includes systemd and dbus-broker. diff --git a/packages/selinux-policy/systems.cil b/packages/selinux-policy/systems.cil index 1e846e090e8..c80126fc0af 100644 --- a/packages/selinux-policy/systems.cil +++ b/packages/selinux-policy/systems.cil @@ -18,5 +18,7 @@ (classmapping systems use (cap_userns (all))) (classmapping systems use (cap2_userns (not (mac_override mac_admin)))) (classmapping systems use (key (all))) +(classmapping systems use (lockdown (all))) +(classmapping systems use (perf_event (all))) (classmapping systems use ( system (not (halt reboot status start stop enable disable reload)))) From e0af06ea21a4deaf9776acccbb45211d729e5b63 Mon Sep 17 00:00:00 2001 From: Ben Cressey Date: Wed, 21 Apr 2021 23:38:25 +0000 Subject: [PATCH 6/7] refactor kmod kit creation Now that the installed kernel is a property of the variant, we need to ensure that the kmod kit includes the development files that match the chosen kernel. By creating it as another stage in the variant build, it's easier to find the right files, and to guarantee that the kmod kit is recreated whenever the image changes. Signed-off-by: Ben Cressey --- Dockerfile | 37 +++++++++++- Makefile.toml | 94 +++-------------------------- tools/buildsys/src/builder.rs | 16 ++++- tools/buildsys/src/builder/error.rs | 6 ++ tools/buildsys/src/manifest.rs | 10 +++ tools/rpm2kmodkit | 46 ++++++++++++++ 6 files changed, 116 insertions(+), 93 deletions(-) create mode 100755 tools/rpm2kmodkit diff --git a/Dockerfile b/Dockerfile index 42788a94499..39c3da8142d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,12 @@ # filesystem at /host. ARG SDK +ARG TOOLCHAIN +ARG ARCH +ARG GOARCH + FROM ${SDK} as sdk +FROM --platform=linux/${GOARCH} ${TOOLCHAIN}-${ARCH} as toolchain ############################################################################################ # Section 1: The following build stages are used to build rpm.spec packages @@ -120,12 +125,16 @@ WORKDIR /root USER root RUN --mount=target=/host \ - mkdir -p /local/rpms /local/migrations ./rpmbuild/RPMS \ + mkdir -p /local/rpms /local/migrations /local/archives ./rpmbuild/RPMS \ && ln -s /host/build/rpms/*.rpm ./rpmbuild/RPMS \ && find /host/build/rpms/ -maxdepth 1 -type f \ -name "bottlerocket-${ARCH}-migrations-*.rpm" \ -not -iname '*debuginfo*' \ -exec cp '{}' '/local/migrations/' ';' \ + && KERNEL="$(printf "%s\n" ${PACKAGES} | awk '/^kernel-/{print $1}')" \ + && find /host/build/rpms/ -maxdepth 1 -type f \ + -name "bottlerocket-${ARCH}-${KERNEL}-archive-*.rpm" \ + -exec cp '{}' '/local/archives/' ';' \ && createrepo_c \ -o ./rpmbuild/RPMS \ -x '*-debuginfo-*.rpm' \ @@ -186,8 +195,30 @@ RUN --mount=target=/host \ && echo ${NOCACHE} # =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= -# Copies the build artifacts (Bottlerocket image files and migrations) to their expected -# location so that buildsys can find them and copy them out. +# Creates an archive of kernel development sources and toolchain. +FROM repobuild as kmodkitbuild +ARG ARCH +ARG VERSION_ID +ARG BUILD_ID +ARG NOCACHE +ARG VARIANT +ENV VARIANT=${VARIANT} VERSION_ID=${VERSION_ID} BUILD_ID=${BUILD_ID} + +USER root +COPY --from=toolchain /toolchain /local/toolchain + +WORKDIR /tmp +RUN --mount=target=/host \ + /host/tools/rpm2kmodkit \ + --archive-dir=/local/archives \ + --toolchain-dir=/local/toolchain \ + --output-dir=/local/output \ + && echo ${NOCACHE} + +# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= +# Copies the build artifacts (Bottlerocket image files, migrations, and kmod kit) to their +# expected location so that buildsys can find them and copy them out. FROM scratch AS variant COPY --from=imgbuild /local/output/* /output/ COPY --from=migrationbuild /local/output/* /output/ +COPY --from=kmodkitbuild /local/output/* /output/ diff --git a/Makefile.toml b/Makefile.toml index f59d26b0104..c183fb052ce 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -114,7 +114,7 @@ PUBLISH_AMI_NAME_DEFAULT = "${BUILDSYS_NAME}-${BUILDSYS_VARIANT}-${BUILDSYS_ARCH # The name of the kmod kit archive, used to ease building out-of-tree kernel modules. BUILDSYS_KMOD_KIT = "${BUILDSYS_VARIANT}-${BUILDSYS_ARCH}-kmod-kit-v${BUILDSYS_VERSION_IMAGE}" -BUILDSYS_KMOD_KIT_PATH="${BUILDSYS_ARCHIVES_DIR}/${BUILDSYS_KMOD_KIT}.tar.xz" +BUILDSYS_KMOD_KIT_PATH="${BUILDSYS_OUTPUT_DIR}/latest/${BUILDSYS_KMOD_KIT}.tar.xz" # The name of the OVA bundle that will be built if the current variant builds VMDK artifacts BUILDSYS_OVA = "${BUILDSYS_NAME_FULL}.ova" @@ -335,92 +335,8 @@ cargo build \ ''' ] -[tasks.build-kernel] -env = { "PACKAGE" = "kernel" } -run_task = "build-package" - -[tasks.build-kmod-kit] -dependencies = ["build-kernel"] -script_runner = "bash" -script = [ -''' -mkdir -p "${BUILDSYS_ARCHIVES_DIR}" - -toolchain="toolchain-${BUILDSYS_SDK_VERSION}.${BUILDSYS_ARCH}.tar.gz" -if [ ! -s "${BUILDSYS_ARCHIVES_DIR}/${toolchain}" ] ; then - if ! docker create --name "${toolchain}" \ - ${BUILDSYS_TOOLCHAIN}-${BUILDSYS_ARCH} true >/dev/null 2>&1 ; then - echo "could not create toolchain container" >&2 - exit 1 - fi - if ! docker cp "${toolchain}":toolchain - \ - | gzip --fast > "${BUILDSYS_ARCHIVES_DIR}/${toolchain}" ; then - echo "could not extract toolchain from container" >&2 - exit 1 - fi - if ! docker rm -f "${toolchain}" >/dev/null 2>&1 ; then - echo "could not remove toolchain container" >&2 - exit 1 - fi -fi - -# Find the most recent kernel archive. If we have more than one, we want the -# last one that was built. -kernel_archive="$(find "${BUILDSYS_PACKAGES_DIR}" \ - -type f -name '*-'"${BUILDSYS_ARCH}"'-kernel-archive-*.rpm' \ - -printf '%T@ %p\n' | sort -r | awk 'NR==1{print $2}')" - -if [ "${?}" -ne 0 ] || [ -z "${kernel_archive}" ] || [ ! -s "${kernel_archive}" ]; then - echo "Unable to find latest kernel archive for ${BUILDSYS_ARCH} in ${BUILDSYS_PACKAGES_DIR}" - exit 1 -fi - -if [ -s "${BUILDSYS_KMOD_KIT_PATH}" ] && [ "${BUILDSYS_KMOD_KIT_PATH}" -nt "${kernel_archive}" ]; then - echo "Existing kmod kit ${BUILDSYS_KMOD_KIT_PATH} is newer than kernel archive ${kernel_archive}; skipping build." - exit 0 -fi - -prepare_kmod_kit=" -set -e -o pipefail - -mkdir -p /tmp/kit/${BUILDSYS_KMOD_KIT} /tmp/extract - -# Retrieve the toolchain and kernel archives. -pushd /tmp/extract >/dev/null -find /tmp/rpms -name "${kernel_archive##*/}" \ - -exec rpm2cpio {} \; | cpio -idmu --quiet -find -name 'kernel-devel.tar.xz' -exec mv {} /tmp/kit/${BUILDSYS_KMOD_KIT} \; -popd >/dev/null - -# Extract them into the same directory. -pushd /tmp/kit/${BUILDSYS_KMOD_KIT} >/dev/null -tar xf kernel-devel.tar.xz -rm kernel-devel.tar.xz -tar xf /tmp/archives/${toolchain} -popd >/dev/null - -# Merge them together into a unified archive. -pushd /tmp/kit >/dev/null -tar cf ${BUILDSYS_KMOD_KIT}.tar ${BUILDSYS_KMOD_KIT} -xz -T0 ${BUILDSYS_KMOD_KIT}.tar -popd >/dev/null - -mv /tmp/kit/${BUILDSYS_KMOD_KIT}.tar.xz /tmp/archives -" - -docker run --rm \ - --network=host \ - --user "$(id -u):$(id -g)" \ - --security-opt label:disable \ - -v "${BUILDSYS_PACKAGES_DIR}":/tmp/rpms \ - -v "${BUILDSYS_ARCHIVES_DIR}":/tmp/archives \ - "${BUILDSYS_SDK_IMAGE}" \ - bash -c "${prepare_kmod_kit}" -''' -] - [tasks.build-archives] -dependencies = ["build-kmod-kit", "build-ova"] +dependencies = ["build-ova"] [tasks.build-variant] dependencies = ["build-tools", "publish-setup"] @@ -532,7 +448,9 @@ docker run --rm \ dependencies = ["fetch"] script = [ ''' -for link in ${BUILDSYS_OUTPUT_DIR}/latest/${BUILDSYS_NAME_VARIANT}*; do +for link in \ + ${BUILDSYS_OUTPUT_DIR}/latest/${BUILDSYS_NAME_VARIANT}* \ + ${BUILDSYS_OUTPUT_DIR}/latest/*-kmod-kit-* ; do if [ -L "${link}" ]; then rm ${link} fi @@ -551,6 +469,8 @@ for artifact in ${BUILDSYS_OUTPUT_DIR}/${BUILDSYS_NAME_FULL}*; do link_name="${file_name/${BUILDSYS_NAME_FULL}/${BUILDSYS_NAME_VARIANT}}" ln -snf "../${file_name}" "${BUILDSYS_OUTPUT_DIR}/latest/${link_name}" done +ln -snf "../${BUILDSYS_NAME_FULL}-kmod-kit.tar.xz" \ + "${BUILDSYS_OUTPUT_DIR}/latest/${BUILDSYS_KMOD_KIT}.tar.xz" ''' ] diff --git a/tools/buildsys/src/builder.rs b/tools/buildsys/src/builder.rs index 101eb8ca3e5..f1dd8ed775b 100644 --- a/tools/buildsys/src/builder.rs +++ b/tools/buildsys/src/builder.rs @@ -19,7 +19,7 @@ use std::path::{Path, PathBuf}; use std::process::Output; use walkdir::{DirEntry, WalkDir}; -use crate::manifest::ImageFormat; +use crate::manifest::{ImageFormat, SupportedArch}; /* There's a bug in BuildKit that can lead to a build failure during parallel @@ -45,8 +45,11 @@ pub(crate) struct PackageBuilder; impl PackageBuilder { /// Build RPMs for the specified package. pub(crate) fn build(package: &str) -> Result { - let arch = getenv("BUILDSYS_ARCH")?; let output_dir: PathBuf = getenv("BUILDSYS_PACKAGES_DIR")?.into(); + let arch = getenv("BUILDSYS_ARCH")?; + let goarch = serde_plain::from_str::(&arch) + .context(error::UnsupportedArch { arch: &arch })? + .goarch(); // We do *not* want to rebuild most packages when the variant changes, because most aren't // affected; packages that care about variant should "echo cargo:rerun-if-env-changed=VAR" @@ -61,6 +64,7 @@ impl PackageBuilder { let mut args = Vec::new(); args.build_arg("PACKAGE", package); args.build_arg("ARCH", &arch); + args.build_arg("GOARCH", &goarch); args.build_arg("VARIANT", variant); args.build_arg("REPO", repo); @@ -89,10 +93,14 @@ impl VariantBuilder { let variant = getenv("BUILDSYS_VARIANT")?; let arch = getenv("BUILDSYS_ARCH")?; + let goarch = serde_plain::from_str::(&arch) + .context(error::UnsupportedArch { arch: &arch })? + .goarch(); let mut args = Vec::new(); args.build_arg("PACKAGES", packages.join(" ")); args.build_arg("ARCH", &arch); + args.build_arg("GOARCH", &goarch); args.build_arg("VARIANT", &variant); args.build_arg("VERSION_ID", getenv("BUILDSYS_VERSION_IMAGE")?); args.build_arg("BUILD_ID", getenv("BUILDSYS_VERSION_BUILD")?); @@ -155,8 +163,9 @@ fn build( let token = &digest[..12]; let tag = format!("{}-{}", tag, token); - // Our SDK image is picked by the external `cargo make` invocation. + // Our SDK and toolchain are picked by the external `cargo make` invocation. let sdk = getenv("BUILDSYS_SDK_IMAGE")?; + let toolchain = getenv("BUILDSYS_TOOLCHAIN")?; // Avoid using a cached layer from a previous build. let nocache = rand::thread_rng().gen::(); @@ -184,6 +193,7 @@ fn build( build.extend(build_args); build.build_arg("SDK", sdk); + build.build_arg("TOOLCHAIN", toolchain); build.build_arg("NOCACHE", nocache.to_string()); // Avoid using a cached layer from a concurrent build in another checkout. build.build_arg("TOKEN", token); diff --git a/tools/buildsys/src/builder/error.rs b/tools/buildsys/src/builder/error.rs index d57f6354633..a6476e939f9 100644 --- a/tools/buildsys/src/builder/error.rs +++ b/tools/buildsys/src/builder/error.rs @@ -52,6 +52,12 @@ pub(crate) enum Error { var: String, source: std::env::VarError, }, + + #[snafu(display("Unsupported architecture '{}'", arch))] + UnsupportedArch { + arch: String, + source: serde_plain::Error, + }, } pub(super) type Result = std::result::Result; diff --git a/tools/buildsys/src/manifest.rs b/tools/buildsys/src/manifest.rs index b8f82c3595e..49d4c341fc4 100644 --- a/tools/buildsys/src/manifest.rs +++ b/tools/buildsys/src/manifest.rs @@ -215,6 +215,16 @@ pub(crate) enum SupportedArch { Aarch64, } +/// Map a Linux architecture into the corresponding Docker architecture. +impl SupportedArch { + pub(crate) fn goarch(&self) -> &'static str { + match self { + SupportedArch::X86_64 => "amd64", + SupportedArch::Aarch64 => "arm64", + } + } +} + #[derive(Deserialize, Debug)] #[serde(rename_all = "kebab-case")] pub(crate) struct ExternalFile { diff --git a/tools/rpm2kmodkit b/tools/rpm2kmodkit new file mode 100755 index 00000000000..1c0c8871239 --- /dev/null +++ b/tools/rpm2kmodkit @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# +# Create an archive of kernel development sources and toolchain. +set -eu -o pipefail + +for opt in "$@"; do + optarg="$(expr "${opt}" : '[^=]*=\(.*\)')" + case "${opt}" in + --archive-dir=*) ARCHIVE_DIR="${optarg}" ;; + --toolchain-dir=*) TOOLCHAIN_DIR="${optarg}" ;; + --output-dir=*) OUTPUT_DIR="${optarg}" ;; + esac +done + +# Use a friendly name for the top-level directory inside the archive. +KMOD_KIT="${VARIANT}-${ARCH}-kmod-kit-v${VERSION_ID}" + +# Use the build ID within the filename, to align with our build's expectations. +KMOD_KIT_FULL="bottlerocket-${VARIANT}-${ARCH}-${VERSION_ID}-${BUILD_ID}-kmod-kit" + +EXTRACT_DIR="$(mktemp -d)" +KIT_DIR="$(mktemp -d)" +mkdir -p "${OUTPUT_DIR}" "${KIT_DIR}/${KMOD_KIT}" + +# Extract any RPMs and find the kernel development archive. +pushd "${EXTRACT_DIR}" >/dev/null +find "${ARCHIVE_DIR}" -type f -name '*.rpm' \ + -exec rpm2cpio {} \; | cpio -idm --quiet +find -name 'kernel-devel.tar.xz' \ + -exec mv {} "${KIT_DIR}/${KMOD_KIT}" \; +popd >/dev/null + +# Extract it and copy in the toolchain. +pushd "${KIT_DIR}/${KMOD_KIT}" >/dev/null +tar xf kernel-devel.tar.xz +rm kernel-devel.tar.xz +cp -a "${TOOLCHAIN_DIR}" toolchain +popd >/dev/null + +# Merge them together into a unified archive. +pushd "${KIT_DIR}" >/dev/null +tar cf "${OUTPUT_DIR}/${KMOD_KIT_FULL}.tar" "${KMOD_KIT}" +xz -T0 "${OUTPUT_DIR}/${KMOD_KIT_FULL}.tar" +popd >/dev/null + +rm -rf "${EXTRACT_DIR}" "${KIT_DIR}" From cfb32fced20a85164f421238672e8ecb059bcb51 Mon Sep 17 00:00:00 2001 From: Ben Cressey Date: Mon, 26 Apr 2021 18:21:45 +0000 Subject: [PATCH 7/7] release: update sysctl defaults to match AL2 In the 5.10 kernel, the settings for `net.ipv4.ip_default_ttl` and `net.ipv4.tcp_wmem` are no longer applied by patching the kernel. Add them to sysctl defaults to keep the same values for both kernels. Signed-off-by: Ben Cressey --- packages/release/release-sysctl.conf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/release/release-sysctl.conf b/packages/release/release-sysctl.conf index cdaa7d9a56c..24d817bc7f9 100644 --- a/packages/release/release-sysctl.conf +++ b/packages/release/release-sysctl.conf @@ -18,6 +18,12 @@ net.ipv6.neigh.default.gc_thresh2 = 15360 net.ipv4.neigh.default.gc_thresh3 = 16384 net.ipv6.neigh.default.gc_thresh3 = 16384 +# Increasing to account for skb structure growth since the 3.4.x kernel series +net.ipv4.tcp_wmem = 4096 20480 4194304 + +# Bumped the default TTL to 255 (maximum) +net.ipv4.ip_default_ttl = 255 + ## Bottlerocket settings ## # Enable IPv4 forwarding for container networking. net.ipv4.conf.all.forwarding = 1