chore: remove unused packages & rework optional module tests (#1806)

Author: mayeut

docker/build_scripts/install-build-packages.sh

@@ -15,18 +15,19 @@ source "${MY_DIR}/build_utils.sh"
 # make sure the corresponding library is added to RUNTIME_DEPS if applicable
 
 if [ "${OS_ID_LIKE}" = "rhel" ]; then
-	COMPILE_DEPS=(bzip2-devel ncurses-devel readline-devel gdbm-devel libpcap-devel xz-devel openssl openssl-devel keyutils-libs-devel krb5-devel libcom_err-devel curl-devel uuid-devel libffi-devel kernel-headers libdb-devel perl-IPC-Cmd)
+	COMPILE_DEPS=(bzip2-devel ncurses-devel readline-devel gdbm-devel xz-devel openssl openssl-devel curl-devel uuid-devel libffi-devel kernel-headers perl-IPC-Cmd)
 	if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
-		COMPILE_DEPS+=(libidn-devel libXft-devel)
+		COMPILE_DEPS+=(libXft-devel)
+		COMPILE_DEPS+=(keyutils-libs-devel krb5-devel libcom_err-devel libidn-devel)  # we rebuild curl
 	elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then
-		COMPILE_DEPS+=(libidn-devel tk-devel)
+		COMPILE_DEPS+=(tk-devel)
 	else
-		COMPILE_DEPS+=(libidn2-devel tk-devel)
+		COMPILE_DEPS+=(tk-devel)
 	fi
 elif [ "${OS_ID_LIKE}" == "debian" ]; then
-	COMPILE_DEPS=(libbz2-dev libncurses-dev libreadline-dev tk-dev libgdbm-dev libdb-dev libpcap-dev liblzma-dev openssl libssl-dev libkeyutils-dev libkrb5-dev comerr-dev libidn2-0-dev libcurl4-openssl-dev uuid-dev libffi-dev linux-headers-generic)
+	COMPILE_DEPS=(libbz2-dev libncurses-dev libreadline-dev tk-dev libgdbm-dev libdb-dev liblzma-dev openssl libssl-dev libcurl4-openssl-dev uuid-dev libffi-dev linux-headers-generic)
 elif [ "${OS_ID_LIKE}" == "alpine" ]; then
-	COMPILE_DEPS=(bzip2-dev ncurses-dev readline-dev tk-dev gdbm-dev libpcap-dev xz-dev openssl openssl-dev keyutils-dev krb5-dev libcom_err libidn-dev curl-dev util-linux-dev libffi-dev linux-headers)
+	COMPILE_DEPS=(bzip2-dev ncurses-dev readline-dev tk-dev gdbm-dev xz-dev openssl openssl-dev curl-dev util-linux-dev libffi-dev linux-headers)
 else
 	echo "Unsupported policy: '${AUDITWHEEL_POLICY}'"
 	exit 1

docker/build_scripts/install-runtime-packages.sh

@@ -46,25 +46,26 @@ fi
 
 # RUNTIME_DEPS: Runtime dependencies. c.f. install-build-packages.sh
 if [ "${OS_ID_LIKE}" == "rhel" ]; then
-	RUNTIME_DEPS=(zlib bzip2 expat ncurses readline gdbm libpcap xz openssl keyutils-libs libkadm5 libcom_err libcurl uuid libffi libdb)
+	RUNTIME_DEPS=(zlib bzip2 expat ncurses readline gdbm xz openssl libcurl uuid libffi)
 	if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
-		RUNTIME_DEPS+=(libidn libXft)
+		RUNTIME_DEPS+=(libXft)
+		RUNTIME_DEPS+=(keyutils-libs libkadm5 libcom_err libidn)  # we rebuild curl
 	elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then
-		RUNTIME_DEPS+=(libidn tk)
+		RUNTIME_DEPS+=(tk)
 	else
-		RUNTIME_DEPS+=(libidn2 tk)
+		RUNTIME_DEPS+=(tk)
 		# for graalpy
 		RUNTIME_DEPS+=(libxcrypt-compat)
 	fi
 elif [ "${OS_ID_LIKE}" == "debian" ]; then
-  RUNTIME_DEPS=(zlib1g libbz2-1.0 libexpat1 libncurses6 libreadline8 tk libgdbm6 libdb5.3 libpcap0.8 liblzma5 libkeyutils1 libkrb5-3 libcom-err2 libidn2-0 libcurl4 uuid)
+  RUNTIME_DEPS=(zlib1g libbz2-1.0 libexpat1 libncurses6 libreadline8 tk libgdbm6 libdb5.3 liblzma5 libcurl4 uuid)
   if [ "${AUDITWHEEL_POLICY}" == "manylinux_2_31" ]; then
   	RUNTIME_DEPS+=(libffi7 libssl1.1)
   else
   	RUNTIME_DEPS+=(libffi8 libssl3)
   fi
 elif [ "${OS_ID_LIKE}" == "alpine" ]; then
-	RUNTIME_DEPS=(zlib bzip2 expat ncurses-libs readline tk gdbm db xz openssl keyutils-libs krb5-libs libcom_err libidn2 libcurl libuuid libffi)
+	RUNTIME_DEPS=(zlib bzip2 expat ncurses-libs readline tk gdbm xz openssl libcurl libuuid libffi)
 else
 	echo "Unsupported policy: '${AUDITWHEEL_POLICY}'"
 	exit 1
@@ -116,7 +117,7 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
 	fi
 	fixup-mirrors
 elif [ "${OS_ID_LIKE}" == "rhel" ]; then
-	BASE_TOOLS+=(glibc-locale-source glibc-langpack-en hardlink hostname libcurl libnsl libxcrypt which)
+	BASE_TOOLS+=(glibc-locale-source glibc-langpack-en gzip hardlink hostname libcurl libnsl libxcrypt which)
 	echo "tsflags=nodocs" >> /etc/dnf/dnf.conf
 	dnf -y upgrade
 	EPEL=epel-release

docker/tests/modules-check.py

@@ -0,0 +1,44 @@
+import unittest
+
+
+class TestModules(unittest.TestCase):
+    def test_sqlite3(self):
+        # Make sure sqlite3 module can be loaded properly and is the manylinux version one
+        # c.f. https://github.com/pypa/manylinux/issues/1030
+        import sqlite3
+
+        print(f"{sqlite3.sqlite_version=}", end=" ", flush=True)
+        assert sqlite3.sqlite_version_info[0:2] >= (3, 50)
+
+    def test_tkinter(self):
+        # Make sure tkinter module can be loaded properly
+        import tkinter as tk
+
+        print(f"{tk.TkVersion=}", end=" ", flush=True)
+        assert tk.TkVersion >= 8.6
+
+    def test_gdbm(self):
+        # depends on libgdbm
+        import dbm.gnu  # noqa: F401
+
+    def test_ndbm(self):
+        # depends on libdb or libgdbm_compat
+        import dbm.ndbm  # noqa: F401
+
+    def test_readline(self):
+        # depends on libreadline
+        import readline  # noqa: F401
+
+    def test_ncurses(self):
+        # depends on libncurses
+        import curses
+
+        print(f"{curses.ncurses_version=}", end=" ", flush=True)
+
+    def test_ctypes(self):
+        # depends on libffi
+        import ctypes  # noqa: F401
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

docker/tests/run_tests.sh

@@ -60,11 +60,8 @@ for PYTHON in /opt/python/*/bin/python; do
 	PYVERS=$(${PYTHON} -c "import sys; print('.'.join(map(str, sys.version_info[:2])))")
 	PY_GIL=$(${PYTHON} -c "import sysconfig; print('t' if sysconfig.get_config_vars().get('Py_GIL_DISABLED', 0) else '')")
 	if [ "${IMPLEMENTATION}" == "cpython" ]; then
-		# Make sure sqlite3 module can be loaded properly and is the manylinux version one
-		# c.f. https://github.com/pypa/manylinux/issues/1030
-		$PYTHON -c 'import sqlite3; print(sqlite3.sqlite_version); assert sqlite3.sqlite_version_info[0:2] >= (3, 50)'
-		# Make sure tkinter module can be loaded properly
-		$PYTHON -c 'import tkinter; print(tkinter.TkVersion); assert tkinter.TkVersion >= 8.6'
+		# check optional modules can be loaded
+		$PYTHON "${MY_DIR}/modules-check.py"
 		# cpython shall be available as python
 		LINK_VERSION=$("python${PYVERS}${PY_GIL}" -VV)
 		REAL_VERSION=$(${PYTHON} -VV)