Add native-deb* targets to build native Debian packages

In continuation of previous openzfs#13451, this commits adds native-deb*
targets to build native debian packages and updates the Github
workflows to build and test native packages.

Ubuntu 18.04 workflow would still build and test the Alien
converted Debian packages due to unmet build and install
dependencies.

Native packages only build with pre-configured paths (see the
dh_auto_configure section in contrib/debian/rules.in). While
building native packages, paths should not be configured. Initial
config flags e.g. '--enable-debug' are replaced in
contrib/debian/rules.in.

Additional packages on top  of existing zfs packages required to
build native packages include debhelper-compat, dh-python, dkms,
po-debconf, python3-all-dev, python3-sphinx.

For release, contrib/debian/changelog should be updated with
release version.

Signed-off-by: Umer Saleem <[email protected]>

.github/workflows/build-dependencies.txt

@@ -6,6 +6,9 @@ bc
 build-essential
 curl
 dbench
+debhelper-compat
+dh-python
+dkms
 fakeroot
 fio
 gdb
@@ -33,12 +36,15 @@ mdadm
 nfs-kernel-server
 pamtester
 parted
+po-debconf
 python3
+python3-all-dev
 python3-cffi
 python3-dev
 python3-packaging
 python3-pip
 python3-setuptools
+python3-sphinx
 rng-tools-debian
 rsync
 samba

.github/workflows/zfs-tests-functional.yml

@@ -32,15 +32,27 @@ jobs:
         ./configure --enable-debug --enable-debuginfo --enable-asan --enable-ubsan
     - name: Make
       run: |
-        make -j$(nproc) --no-print-directory --silent pkg-utils pkg-kmod
+        # Build Alien converted Debian packages for Ubuntu 18.04 due to unmet dependencies
+        if [ "${{ matrix.os }}" = "18.04" ]; then
+          make -j$(nproc) --no-print-directory --silent pkg-utils pkg-kmod
+        else
+          make --no-print-directory --silent native-deb-utils native-deb-kmod
+          mv ../*.deb .
+          rm ./openzfs-zfs-dkms*.deb ./openzfs-zfs-dracut*.deb
+        fi
     - name: Install
       run: |
-        sudo dpkg -i *.deb
         # Update order of directories to search for modules, otherwise
         #   Ubuntu will load kernel-shipped ones.
         sudo sed -i.bak 's/updates/extra updates/' /etc/depmod.d/ubuntu.conf
+        sudo dpkg -i *.deb
         sudo depmod
         sudo modprobe zfs
+        # Native Debian packages enable and start the services
+        # Stop zfs-zed daemon, as it may interfere with some ZTS test cases
+        if [ "${{ matrix.os }}" != "18.04" ]; then
+          sudo systemctl stop zfs-zed
+        fi
         # Workaround for cloud-init bug
         # see https://github.com/openzfs/zfs/issues/12644
         FILE=/lib/udev/rules.d/10-cloud-init-hook-hotplug.rules

.github/workflows/zfs-tests-sanity.yml

@@ -28,15 +28,19 @@ jobs:
         ./configure --enable-debug --enable-debuginfo --enable-asan --enable-ubsan
     - name: Make
       run: |
-        make -j$(nproc) --no-print-directory --silent pkg-utils pkg-kmod
+        make --no-print-directory --silent native-deb-utils native-deb-kmod
+        mv ../*.deb .
+        rm ./openzfs-zfs-dkms*.deb ./openzfs-zfs-dracut*.deb
     - name: Install
       run: |
-        sudo dpkg -i *.deb
         # Update order of directories to search for modules, otherwise
         #   Ubuntu will load kernel-shipped ones.
         sudo sed -i.bak 's/updates/extra updates/' /etc/depmod.d/ubuntu.conf
+        sudo dpkg -i *.deb
         sudo depmod
         sudo modprobe zfs
+        # Stop default zfs-zed daemon, it may interfere with some ZTS test cases
+        sudo systemctl stop zfs-zed
         # Workaround for cloud-init bug
         # see https://github.com/openzfs/zfs/issues/12644
         FILE=/lib/udev/rules.d/10-cloud-init-hook-hotplug.rules

config/deb.am

@@ -1,14 +1,17 @@
-PHONY += deb-kmod deb-dkms deb-utils deb deb-local
+PHONY += deb-kmod deb-dkms deb-utils deb deb-local native-deb-local \
+	native-deb-utils native-deb-kmod native-deb
 
-deb-local:
+native-deb-local:
 	@(if test "${HAVE_DPKGBUILD}" = "no"; then \
 		echo -e "\n" \
 	"*** Required util ${DPKGBUILD} missing.  Please install the\n" \
 	"*** package for your distribution which provides ${DPKGBUILD},\n" \
 	"*** re-run configure, and try again.\n"; \
 		exit 1; \
-	fi; \
-	if test "${HAVE_ALIEN}" = "no"; then \
+	fi)
+
+deb-local: native-deb-local
+	@(if test "${HAVE_ALIEN}" = "no"; then \
 		echo -e "\n" \
 	"*** Required util ${ALIEN} missing.  Please install the\n" \
 	"*** package for your distribution which provides ${ALIEN},\n" \
@@ -85,3 +88,17 @@ deb-utils: deb-local rpm-utils-initramfs
 	    $$pkg8 $$pkg9 $$pkg10 $$pkg11;
 
 deb: deb-kmod deb-dkms deb-utils
+
+copy-deb:
+	if ! test -d debian; then cp -r contrib/debian debian; chmod +x debian/rules; fi;
+
+native-deb-utils: native-deb-local copy-deb
+	if test "x${ASAN_ENABLED}" = "xyes" ; then export DEB_LDFLAGS_APPEND=-static-libasan DEB_CFLAGS_APPEND=-static-libasan ; fi; \
+	cp contrib/debian/control debian/control; \
+	$(DPKGBUILD) -b -rfakeroot -us -uc;
+
+native-deb-kmod: native-deb-local copy-deb
+	sh scripts/make_gitrev.sh; \
+	fakeroot debian/rules override_dh_binary-modules;
+
+native-deb: native-deb-utils native-deb-kmod

config/zfs-build.m4

@@ -464,6 +464,7 @@ AC_DEFUN([ZFS_AC_DPKG], [
 	AC_SUBST(HAVE_DPKGBUILD)
 	AC_SUBST(DPKGBUILD)
 	AC_SUBST(DPKGBUILD_VERSION)
+	AC_SUBST([CFGOPTS], ["$CFGOPTS"])
 ])
 
 dnl #

configure.ac

@@ -32,6 +32,7 @@
 
 AC_INIT(m4_esyscmd(awk '/^Name:/ {printf $2}' META),
 	m4_esyscmd(awk '/^Version:/ {printf $2}' META))
+CFGOPTS="$*"
 AC_LANG(C)
 ZFS_AC_META
 AC_CONFIG_AUX_DIR([config])
@@ -65,6 +66,7 @@ ZFS_AC_DEBUG_KMEM_TRACKING
 ZFS_AC_DEBUG_INVARIANTS
 
 AC_CONFIG_FILES([
+	contrib/debian/rules
 	Makefile
 	include/Makefile
 	lib/libzfs/libzfs.pc

contrib/debian/.gitignore

@@ -0,0 +1 @@
+rules

contrib/debian/control

@@ -2,8 +2,7 @@ Source: openzfs-linux
 Section: contrib/kernel
 Priority: optional
 Maintainer: ZFS on Linux specific mailing list <[email protected]>
-Build-Depends: abigail-tools,
-               debhelper-compat (= 12),
+Build-Depends: debhelper-compat (= 12),
                dh-python,
                dkms (>> 2.1.1.2-5),
                libaio-dev,

contrib/debian/openzfs-zfs-zed.postinst

@@ -4,13 +4,6 @@ set -e
 zedd="/usr/lib/zfs-linux/zed.d"
 etcd="/etc/zfs/zed.d"
 
-# enable all default zedlets that are not overridden
-while read -r file ; do
-    etcfile="${etcd}/${file}"
-    [ -e "${etcfile}" ] && continue
-    ln -sfT "${zedd}/${file}" "${etcfile}"
-done < "${zedd}/DEFAULT-ENABLED"
-
 # remove the overrides created in prerm
 find "${etcd}" -maxdepth 1 -lname '/dev/null' -delete
 # remove any dangling symlinks to old zedlets

contrib/debian/openzfs-zfsutils.install

@@ -131,5 +131,4 @@ usr/share/man/man8/zstreamdump.8
 usr/share/man/man4/spl.4
 usr/share/man/man4/zfs.4
 usr/share/man/man7/zpool-features.7
-usr/share/man/man7/dracut.zfs.7
 usr/share/man/man8/zpool_influxdb.8

contrib/debian/rules → contrib/debian/rules.in

@@ -35,7 +35,7 @@ override_dh_autoreconf:
 
 override_dh_auto_configure:
 	@# Build the userland, but don't build the kernel modules.
-	dh_auto_configure -- \
+	dh_auto_configure -- @CFGOPTS@ \
 	  --bindir=/usr/bin \
 	  --sbindir=/sbin \
 	  --libdir=/lib/"$(DEB_HOST_MULTIARCH)" \
@@ -195,7 +195,7 @@ override_dh_prep-deb-files:
 
 override_dh_configure_modules: override_dh_configure_modules_stamp
 override_dh_configure_modules_stamp:
-	./configure \
+	./configure @CFGOPTS@ \
 		--with-config=kernel \
 		--with-linux=$(KSRC) \
 		--with-linux-obj=$(KOBJ)