python 3.13 rebuild: part 7

.github/workflows/package_updates.yml

@@ -113,13 +113,15 @@ jobs:
           size: 16G
           priority: 100
           device_name: /dev/zram0
+      - name: Load Docker image
+        run: |
+          ./scripts/run-docker.sh echo ""
       - name: Free additional disk space
-        run: CLEAN_DOCKER_IMAGES=false ./scripts/free-space.sh
+        run: ./scripts/free-space.sh
       - name: Process package updates
         env:
           GITHUB_TOKEN: ${{ secrets.TERMUXBOT2_TOKEN }}
           BUILD_PACKAGES: "true"
-          TERMUX_DOCKER__CONTAINER_EXEC_COMMAND__PRE_CHECK_IF_WILL_BUILD_PACKAGES: "true"
           CREATE_ISSUE: "true"
           GIT_COMMIT_PACKAGES: "true"
           GIT_PUSH_PACKAGES: "true"

.github/workflows/packages.yml

@@ -20,6 +20,10 @@ on:
       packages:
         description: "A space-separated names of packages selected for rebuilding"
         required: true
+      free-space:
+        description: "Free space even if not building large package (useful when building a large number of packages)"
+        type: boolean
+        default: false
 
 permissions: {} # none
 
@@ -98,7 +102,6 @@ jobs:
           # Forces CI to cancel current build with status 'passed'
           if grep -qiP '^\s*%ci:no-build\s*$' <(git log --format="%B" -n 1 --no-merges "HEAD"); then
             tar cf artifacts/debs-${{ matrix.target_arch }}.tar debs
-            echo "docker-build=true" >> $GITHUB_OUTPUT
             echo "[!] Force exiting as tag '%ci:no-build' was applied to HEAD commit message."
             exit 0
           fi
@@ -186,31 +189,34 @@ jobs:
 
         echo "packages: ${packages[*]}"
 
-        docker='true'
+        free_space='false'
+        if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+          free_space=${{ github.event.inputs.free-space }}
+        else
+          if grep -qiP '^\s*%ci:free-disk\s*$' <(git log --format="%B" -n 1 --no-merges "HEAD"); then
+            echo "[!] Force exiting as tag '%ci:free-disk' was applied to HEAD commit message."
+            exit 0
+          fi
+        fi
         if [[ "${#packages[@]}" -gt 0 ]]; then
           for pkg in "${packages[@]}"; do
             if grep -qFx "$pkg" ./scripts/big-pkgs.list; then
-              docker='false'
+              free_space='true'
               break
             fi
           done
         fi
+        echo "free-space=$free_space" >> $GITHUB_OUTPUT
 
-        echo "docker-build=$docker" >> $GITHUB_OUTPUT
+        needs_docker_build=false
         if [ "${{ github.event_name }}" != "workflow_dispatch" ]; then
           # Build local Docker image if setup scripts were changed.
           # Useful for pull requests submitting changes for both build environment and packages.
           if grep -qP '^scripts/(Dockerfile|properties\.sh|setup-android-sdk\.sh|setup-ubuntu\.sh)$' <<< "$CHANGED_FILES"; then
-            echo "Detected changes for environment setup scripts. Building custom Docker image now."
-            if [ $docker == 'false' ]; then
-              echo "Skipping due to building large packages."
-              exit 0
-            fi
-            cd ./scripts
-            docker build -t ghcr.io/termux/package-builder:latest .
-            cd ..
+            needs_docker_build=true
           fi
         fi
+        echo "needs-docker-build=$needs_docker_build" >> $GITHUB_OUTPUT
 
     - name: Lint packages
       run: |
@@ -235,22 +241,24 @@ jobs:
         priority: 100
         device_name: /dev/zram0
 
+    - name: Build docker image
+      if: ${{ steps.build-info.needs-docker-build == 'true' }}
+      run: |
+        docker build -t ghcr.io/termux/package-builder:latest scripts/
+        docker buildx prune -af
+
+    - name: Load Docker image
+      if: ${{ steps.build-info.outputs.free-space == 'true' && steps.build-info.outputs.skip-building != 'true' }}
+      run: |
+        ./scripts/run-docker.sh echo ""
+
     - name: Free additional disk space (if needed)
-      if: ${{ steps.build-info.outputs.docker-build == 'false' && steps.build-info.outputs.skip-building != 'true' }}
+      if: ${{ steps.build-info.outputs.free-space == 'true' && steps.build-info.outputs.skip-building != 'true' }}
       run: |
-        ./scripts/setup-ubuntu.sh
-        # need to unset these for setup-android-sdk.sh.
-        unset NDK ANDROID_HOME
-        ./scripts/setup-android-sdk.sh
-        rm -f ${HOME}/lib/ndk-*.zip ${HOME}/lib/sdk-*.zip
-        sudo apt install ninja-build
         ./scripts/free-space.sh
 
     - name: Build packages
       if: ${{ steps.build-info.outputs.skip-building != 'true' }}
-      env:
-        DOCKER_BUILD: ${{ steps.build-info.outputs.docker-build }}
-        TERMUX_DOCKER__CONTAINER_EXEC_COMMAND__PRE_CHECK_IF_WILL_BUILD_PACKAGES: "true"
       run: |
         declare -a packages=()
         for repo_path in $(jq --raw-output 'del(.pkg_format) | keys | .[]' repo.json); do
@@ -263,16 +271,7 @@ jobs:
         echo "packages: ${packages[*]}"
 
         if [[ "${#packages[@]}" -gt 0 ]]; then
-          if [ "$DOCKER_BUILD" == 'false' ]; then
-            # these need to be unset a second time again for ./build-package.sh
-            # when it is run outside of Docker, because GitHub Actions does not
-            # support permanently unsetting variables at time of writing.
-            # https://github.com/actions/runner/issues/1126
-            unset NDK ANDROID_HOME
-            ./build-package.sh -I -C -a "${{ matrix.target_arch }}" "${packages[@]}"
-          else
-            ./scripts/run-docker.sh ./build-package.sh -I -C -a "${{ matrix.target_arch }}" "${packages[@]}"
-          fi
+          ./scripts/run-docker.sh -d ./build-package.sh -I -C -a "${{ matrix.target_arch }}" "${packages[@]}"
         fi
 
     - name: Generate build artifacts
@@ -317,6 +316,10 @@ jobs:
       with:
         name: debs-${{ matrix.target_arch }}-${{ github.sha }}
         path: ./artifacts
+    - name: AppArmor Logs
+      if: always()
+      run: |
+        sudo dmesg | grep apparmor
 
   test-buildorder-random:
     permissions:

CODEOWNERS

@@ -15,6 +15,10 @@
 /scripts/ @Grimler91 @thunder-coding
 /repo.json @Grimler91 @thunder-coding
 
+# Docker security profiles
+/scripts/profile.json @thunder-coding @licy183
+/scripts/*.apparmor @thunder-coding
+
 # Build script linter
 /scripts/lint-packages.sh @TomJo2000
 

build-package.sh

@@ -149,6 +149,10 @@ source "$TERMUX_SCRIPTDIR/scripts/build/setup/termux_setup_ldc.sh"
 # shellcheck source=scripts/build/setup/termux_setup_no_integrated_as.sh
 source "$TERMUX_SCRIPTDIR/scripts/build/setup/termux_setup_no_integrated_as.sh"
 
+# Utility function for setting up build-python for cross-compilation of Python and crossenv
+# shellcheck source=scripts/build/setup/termux_setup_build_python.sh
+source "$TERMUX_SCRIPTDIR/scripts/build/setup/termux_setup_build_python.sh"
+
 # Utility function for python packages to setup a python.
 # shellcheck source=scripts/build/setup/termux_setup_python_pip.sh
 source "$TERMUX_SCRIPTDIR/scripts/build/setup/termux_setup_python_pip.sh"

clean.sh

@@ -93,5 +93,14 @@ fi
 		rm -Rf "/data/data/.built-packages"
 	fi
 
-	rm -Rf "$TERMUX_TOPDIR"
+	# unmount overlayfs before we remove the parent directory
+	[ -d "$TERMUX_TOPDIR" ] && for dir in $(find "$TERMUX_TOPDIR" -type d); do
+		if mountpoint -q "$dir"; then
+			umount "$dir"
+		fi
+	done
+
+	# We can't use rm -Rf "$TERMUX_TOPDIR" in case the "$TERMUX_TOPDIR" is mounted as a Docker volume
+	find "$TERMUX_TOPDIR" -type f,l -delete
+	find "$TERMUX_TOPDIR" -type d ! -path "$TERMUX_TOPDIR" -delete
 } 5< "$TERMUX_BUILD_LOCK_FILE"

packages/blueprint-compiler/build.sh

@@ -3,6 +3,7 @@ TERMUX_PKG_DESCRIPTION="Markup language for GTK user interfaces"
 TERMUX_PKG_LICENSE="LGPL-3.0"
 TERMUX_PKG_MAINTAINER="@termux"
 TERMUX_PKG_VERSION="0.20.0"
+TERMUX_PKG_REVISION=1
 TERMUX_PKG_SRCURL=https://download.gnome.org/sources/blueprint-compiler/${TERMUX_PKG_VERSION%.*}/blueprint-compiler-${TERMUX_PKG_VERSION}.tar.xz
 TERMUX_PKG_SHA256=ec786d66f583e8296c845f1f82834d27b369f39d55a6380b34880493e22db382
 TERMUX_PKG_AUTO_UPDATE=true

packages/pycairo/build.sh

@@ -3,6 +3,7 @@ TERMUX_PKG_DESCRIPTION="Python bindings for the cairo graphics library"
 TERMUX_PKG_LICENSE="LGPL-2.1"
 TERMUX_PKG_MAINTAINER="@termux"
 TERMUX_PKG_VERSION="1.29.0"
+TERMUX_PKG_REVISION=1
 TERMUX_PKG_SRCURL=https://github.com/pygobject/pycairo/releases/download/v${TERMUX_PKG_VERSION}/pycairo-${TERMUX_PKG_VERSION}.tar.gz
 TERMUX_PKG_SHA256=f3f7fde97325cae80224c09f12564ef58d0d0f655da0e3b040f5807bd5bd3142
 TERMUX_PKG_AUTO_UPDATE=true

packages/pygobject/build.sh

@@ -3,6 +3,7 @@ TERMUX_PKG_DESCRIPTION="Python package which provides bindings for GObject based
 TERMUX_PKG_LICENSE="LGPL-2.1"
 TERMUX_PKG_MAINTAINER="@termux"
 TERMUX_PKG_VERSION="3.56.0"
+TERMUX_PKG_REVISION=1
 TERMUX_PKG_SRCURL=https://download.gnome.org/sources/pygobject/${TERMUX_PKG_VERSION%.*}/pygobject-${TERMUX_PKG_VERSION}.tar.gz
 TERMUX_PKG_SHA256=4fbb5bf47524e01026f8e309dd54233eb0f75f2281392c5bf0df5d9041cc7891
 TERMUX_PKG_AUTO_UPDATE=true

packages/python-pip/android-api-abi.patch

@@ -0,0 +1,14 @@
+Allow installing without specifying abi version. Needed for crossenv specifically.
+May not be needed for on-device, but let's just keep it just in case in case there is some psycho who is doing this already
+diff --git a/src/pip/_vendor/packaging/tags.py b/src/pip/_vendor/packaging/tags.py
+index 8522f59c4..ef55342fb 100644
+--- a/src/pip/_vendor/packaging/tags.py
++++ b/src/pip/_vendor/packaging/tags.py
+@@ -563,6 +563,7 @@ def android_platforms(
+     # without major patching. Yield every API level from the maximum down to the
+     # minimum, inclusive.
+     min_api_level = 16
++    yield f"android_{abi}"
+     for ver in range(api_level, min_api_level - 1, -1):
+         yield f"android_{ver}_{abi}"
+

packages/python/0001-fix-hardcoded-paths.patch

@@ -1,26 +1,3 @@
---- a/Lib/aifc.py
-+++ b/Lib/aifc.py
-@@ -920,7 +920,7 @@
- if __name__ == '__main__':
-     import sys
-     if not sys.argv[1:]:
--        sys.argv.append('/usr/demos/data/audio/bach.aiff')
-+        sys.argv.append('@TERMUX_PREFIX@/demos/data/audio/bach.aiff')
-     fn = sys.argv[1]
-     with open(fn, 'r') as f:
-         print("Reading", fn)
---- a/Lib/mailcap.py
-+++ b/Lib/mailcap.py
-@@ -55,7 +55,8 @@
-             # Don't bother with getpwuid()
-             home = '.' # Last resort
-         mailcaps = [home + '/.mailcap', '/etc/mailcap',
--                '/usr/etc/mailcap', '/usr/local/etc/mailcap']
-+                '/usr/etc/mailcap', '/usr/local/etc/mailcap',
-+                '@TERMUX_PREFIX@/etc/mailcap']
-     return mailcaps
-
-
 --- a/Lib/mimetypes.py
 +++ b/Lib/mimetypes.py
 @@ -49,6 +49,7 @@

packages/python/0005-impl-multiprocessing.patch

@@ -10,18 +10,6 @@
              _dir_candidates = []
 
 
---- a/Modules/_multiprocessing/multiprocessing.c
-+++ b/Modules/_multiprocessing/multiprocessing.c
-@@ -172,7 +172,7 @@
-     _MULTIPROCESSING_RECV_METHODDEF
-     _MULTIPROCESSING_SEND_METHODDEF
- #endif
--#if !defined(POSIX_SEMAPHORES_NOT_ENABLED) && !defined(__ANDROID__)
-+#if !defined(POSIX_SEMAPHORES_NOT_ENABLED)
-     _MULTIPROCESSING_SEM_UNLINK_METHODDEF
- #endif
-     {NULL}
-
 --- a/Modules/_multiprocessing/posixshmem.c
 +++ b/Modules/_multiprocessing/posixshmem.c
 @@ -11,6 +11,72 @@

packages/python/0006-disable-multiarch.patch

@@ -0,0 +1,12 @@
+diff --git a/configure.ac b/configure.ac
+index 597a44b331a..fa78e09c192 100644
+--- a/configure.ac
++++ b/configure.ac
+@@ -1134,6 +1134,7 @@ dnl architecture. PLATFORM_TRIPLET will be a pair or single value for these
+ dnl platforms.
+ AC_MSG_CHECKING([for multiarch])
+ AS_CASE([$ac_sys_system],
++  [Linux-android], [MULTIARCH=""],
+   [Darwin*], [MULTIARCH=""],
+   [iOS], [MULTIARCH=""],
+   [FreeBSD*], [MULTIARCH=""],

packages/python/0008-fix-pkgconfig-variable-substitution.patch

@@ -0,0 +1,16 @@
+Needed for proper substitution of @LIBPYTHON@ in python3.pc.in
+
+The former style of substitution is fine in Makefiles where variables are expanded, not in static generated files.
+diff --git a/configure.ac b/configure.ac
+index 043ec957f40..53bed63310e 100644
+--- a/configure.ac
++++ b/configure.ac
+@@ -6424,7 +6424,7 @@ LIBPYTHON=''
+ # On Android and Cygwin the shared libraries must be linked with libpython.
+ if test "$PY_ENABLE_SHARED" = "1" && ( test -n "$ANDROID_API_LEVEL" || test "$MACHDEP" = "cygwin"); then
+   MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(LDLIBRARY)"
+-  LIBPYTHON="\$(BLDLIBRARY)"
++  LIBPYTHON="$(BLDLIBRARY)"
+ fi
+
+ # On iOS the shared libraries must be linked with the Python framework

packages/python/0009-fix-ctypes-util-find_library.patch

@@ -0,0 +1,22 @@
+This was introduced in https://github.com/python/cpython/commit/1f8b24ef69896680d6ba6005e75e1cc79a744f9e but breaks our builds as we need directories from other paths as well
+
+diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py
+index 117bf06cb01..12d7428fe9a 100644
+--- a/Lib/ctypes/util.py
++++ b/Lib/ctypes/util.py
+@@ -89,15 +89,6 @@ def find_library(name):
+
+     from ctypes._aix import find_library
+
+-elif sys.platform == "android":
+-    def find_library(name):
+-        directory = "/system/lib"
+-        if "64" in os.uname().machine:
+-            directory += "64"
+-
+-        fname = f"{directory}/lib{name}.so"
+-        return fname if os.path.isfile(fname) else None
+-
+ elif os.name == "posix":
+     # Andreas Degert's find functions, using gcc, /sbin/ldconfig, objdump
+     import re, tempfile

packages/python/0011-do-not-hardlink.patch → packages/python/0010-do-not-hardlink.patch

@@ -1,15 +1,17 @@
+diff --git a/Makefile.pre.in b/Makefile.pre.in
+index a7dc9709d62..d6b84dc8905 100644
 --- a/Makefile.pre.in
 +++ b/Makefile.pre.in
-@@ -817,7 +817,7 @@ $(LIBRARY): $(LIBRARY_OBJS)
- libpython$(LDVERSION).so: $(LIBRARY_OBJS) $(DTRACE_OBJS)
- 	if test $(INSTSONAME) != $(LDLIBRARY); then \
- 		$(BLDSHARED) -Wl,-h$(INSTSONAME) -o $(INSTSONAME) $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \
+@@ -917,7 +917,7 @@ libpython$(LDVERSION).so: $(LIBRARY_OBJS) $(DTRACE_OBJS)
+ 		$(BLDSHARED) -o $@ $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \
+ 	fi
+ 	if test $(INSTSONAME) != $@; then \
 -		$(LN) -f $(INSTSONAME) $@; \
 +		$(LN) -sf $(INSTSONAME) $@; \
- 	else \
- 		$(BLDSHARED) -o $@ $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \
  	fi
-@@ -1971,7 +1971,7 @@ altbininstall: $(BUILDPYTHON) @FRAMEWORKPYTHONW@
+
+ libpython3.so:	libpython$(LDVERSION).so
+@@ -2202,7 +2202,7 @@ altbininstall: $(BUILDPYTHON) @FRAMEWORKPYTHONW@
  		if test -f $(DESTDIR)$(BINDIR)/python$(VERSION)$(EXE) -o -h $(DESTDIR)$(BINDIR)/python$(VERSION)$(EXE); \
  		then rm -f $(DESTDIR)$(BINDIR)/python$(VERSION)$(EXE); \
  		fi; \