From 3e0d9b231eb5751ca86ff953a906fc03a2ec2719 Mon Sep 17 00:00:00 2001 From: Geoffrey Thomas Date: Mon, 4 Aug 2025 13:41:33 -0400 Subject: [PATCH 1/8] .github/workflows: Fix quoting around labels for ci-matrix --- .github/workflows/linux.yml | 2 +- .github/workflows/macos.yml | 2 +- .github/workflows/windows.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 2ed390b7..0e2b2ce4 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -173,7 +173,7 @@ jobs: run: | uv run ci-matrix.py \ --platform linux \ - --labels '${STEPS_GET_LABELS_OUTPUTS_LABELS}' \ + --labels "${STEPS_GET_LABELS_OUTPUTS_LABELS}" \ --max-shards 2 \ ${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} \ > matrix.json diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 0bc73b13..02ce471d 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -90,7 +90,7 @@ jobs: - name: Generate build matrix id: set-matrix run: | - uv run ci-matrix.py --platform darwin --labels '${STEPS_GET_LABELS_OUTPUTS_LABELS}' ${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} > matrix.json + uv run ci-matrix.py --platform darwin --labels "${STEPS_GET_LABELS_OUTPUTS_LABELS}" ${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} > matrix.json # Extract python-build matrix echo "matrix=$(jq -c '."python-build"' matrix.json)" >> $GITHUB_OUTPUT diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index e1fea413..95eacef8 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -90,7 +90,7 @@ jobs: - name: Generate build matrix id: set-matrix run: | - uv run ci-matrix.py --platform windows --labels '${STEPS_GET_LABELS_OUTPUTS_LABELS}' ${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} > matrix.json + uv run ci-matrix.py --platform windows --labels "${STEPS_GET_LABELS_OUTPUTS_LABELS}" ${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} > matrix.json # Extract python-build matrix echo "matrix=$(jq -c '."python-build"' matrix.json)" >> $GITHUB_OUTPUT From 8a85bae13de2e9c857672bdf62a211faff2d2c08 Mon Sep 17 00:00:00 2001 From: Geoffrey Thomas Date: Mon, 4 Aug 2025 09:00:20 -0400 Subject: [PATCH 2/8] Remove Tix (#723) --- cpython-unix/Makefile | 11 ------ cpython-unix/build-cpython.sh | 6 +--- cpython-unix/build-main.py | 2 +- cpython-unix/build.py | 56 ------------------------------ cpython-unix/extension-modules.yml | 2 -- cpython-unix/targets.yml | 16 --------- docs/distributions.rst | 4 +-- docs/quirks.rst | 23 ++++++------ docs/status.rst | 2 +- 9 files changed, 18 insertions(+), 104 deletions(-) diff --git a/cpython-unix/Makefile b/cpython-unix/Makefile index 40c34a27..d3164616 100644 --- a/cpython-unix/Makefile +++ b/cpython-unix/Makefile @@ -195,16 +195,6 @@ $(OUTDIR)/sqlite-$(SQLITE_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(OUTDIR)/tcl-$(TCL_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-tcl.sh $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) tcl -TIX_DEPENDS = \ - $(HERE)/build-tix.sh \ - $(OUTDIR)/tcl-$(TCL_VERSION)-$(PACKAGE_SUFFIX).tar \ - $(OUTDIR)/tk-$(TK_VERSION)-$(PACKAGE_SUFFIX).tar \ - $(if $(NEED_LIBX11),$(OUTDIR)/libX11-$(LIBX11_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(NULL) - -$(OUTDIR)/tix-$(TIX_VERSION)-$(PACKAGE_SUFFIX).tar: $(TIX_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) tix - TK_DEPENDS = \ $(HOST_PYTHON_DEPENDS) \ $(HERE)/build-tk.sh \ @@ -271,7 +261,6 @@ PYTHON_DEPENDS_$(1) := \ $$(if $$(NEED_SQLITE),$$(OUTDIR)/sqlite-$$(SQLITE_VERSION)-$$(PACKAGE_SUFFIX).tar) \ $$(if $$(NEED_TCL),$$(OUTDIR)/tcl-$$(TCL_VERSION)-$$(PACKAGE_SUFFIX).tar) \ $$(if $$(NEED_TK),$$(OUTDIR)/tk-$$(TK_VERSION)-$$(PACKAGE_SUFFIX).tar) \ - $$(if $$(NEED_TIX),$$(OUTDIR)/tix-$$(TIX_VERSION)-$$(PACKAGE_SUFFIX).tar) \ $$(if $$(NEED_UUID),$$(OUTDIR)/uuid-$$(UUID_VERSION)-$$(PACKAGE_SUFFIX).tar) \ $$(if $$(NEED_XZ),$$(OUTDIR)/xz-$$(XZ_VERSION)-$$(PACKAGE_SUFFIX).tar) \ $$(if $$(NEED_ZLIB),$$(OUTDIR)/zlib-$$(ZLIB_VERSION)-$$(PACKAGE_SUFFIX).tar) \ diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index cc93f0ad..e28c70bc 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -1244,16 +1244,12 @@ fi rm -f ${ROOT}/out/python/build/lib/{libdb-6.0,libxcb-*,libX11-xcb}.a if [ -d "${TOOLS_PATH}/deps/lib/tcl8" ]; then - # Copy tcl/tk/tix resources needed by tkinter. + # Copy tcl/tk resources needed by tkinter. mkdir ${ROOT}/out/python/install/lib/tcl # Keep this list in sync with tcl_library_paths. for source in ${TOOLS_PATH}/deps/lib/{itcl4.2.4,tcl8,tcl8.6,thread2.8.9,tk8.6}; do cp -av $source ${ROOT}/out/python/install/lib/ done - - if [[ "${PYBUILD_PLATFORM}" != macos* ]]; then - cp -av ${TOOLS_PATH}/deps/lib/Tix8.4.3 ${ROOT}/out/python/install/lib/ - fi fi # Copy the terminfo database if present. diff --git a/cpython-unix/build-main.py b/cpython-unix/build-main.py index 7a7d08b1..23d473ef 100755 --- a/cpython-unix/build-main.py +++ b/cpython-unix/build-main.py @@ -182,7 +182,7 @@ def main(): # because we can get some speedup from parallel operations. But we also don't # share a make job server with each build. So if we didn't limit the # parallelism we could easily oversaturate the CPU. Higher levels of - # parallelism don't result in meaningful build speedups because tk/tix has + # parallelism don't result in meaningful build speedups because tk has # a long, serial dependency chain that can't be built in parallel. parallelism = min(1 if args.serial else 4, multiprocessing.cpu_count()) diff --git a/cpython-unix/build.py b/cpython-unix/build.py index b35d1721..25976ce5 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -379,48 +379,6 @@ def build_libedit( build_env.get_tools_archive(dest_archive, "deps") -def build_tix( - settings, client, image, host_platform, target_triple, build_options, dest_archive -): - tcl_archive = download_entry("tcl", DOWNLOADS_PATH) - tk_archive = download_entry("tk", DOWNLOADS_PATH) - tix_archive = download_entry("tix", DOWNLOADS_PATH) - - with build_environment(client, image) as build_env: - if settings.get("needs_toolchain"): - build_env.install_toolchain( - BUILD, - host_platform, - target_triple, - binutils=install_binutils(host_platform), - clang=True, - musl="musl" in target_triple, - static="static" in build_options, - ) - - depends = {"tcl", "tk"} - if not host_platform.startswith("macos_"): - depends |= {"libX11", "xorgproto"} - - for p in sorted(depends): - build_env.install_artifact_archive(BUILD, p, target_triple, build_options) - - for p in (tcl_archive, tk_archive, tix_archive, SUPPORT / "build-tix.sh"): - build_env.copy_file(p) - - env = { - "TOOLCHAIN": "clang-%s" % host_platform, - "TCL_VERSION": DOWNLOADS["tcl"]["version"], - "TIX_VERSION": DOWNLOADS["tix"]["version"], - "TK_VERSION": DOWNLOADS["tk"]["version"], - } - - add_target_env(env, host_platform, target_triple, build_env) - - build_env.run("build-tix.sh", environment=env) - build_env.get_tools_archive(dest_archive, "deps") - - def build_cpython_host( client, image, @@ -946,9 +904,6 @@ def build_cpython( "tk8.6", ] - if "-apple" not in target_triple: - python_info["tcl_library_paths"].append("Tix8.4.3") - if "-apple" in target_triple: python_info["apple_sdk_platform"] = env["APPLE_SDK_PLATFORM"] python_info["apple_sdk_version"] = env["APPLE_SDK_VERSION"] @@ -1239,17 +1194,6 @@ def main(): python_host_version=python_host_version, ) - elif action == "tix": - build_tix( - settings, - client, - get_image(client, ROOT, BUILD, docker_image, host_platform), - host_platform=host_platform, - target_triple=target_triple, - build_options=build_options, - dest_archive=dest_archive, - ) - elif action == "tk": extra_archives = {"tcl"} if not host_platform.startswith("macos_"): diff --git a/cpython-unix/extension-modules.yml b/cpython-unix/extension-modules.yml index c069ca1b..8f74aa55 100644 --- a/cpython-unix/extension-modules.yml +++ b/cpython-unix/extension-modules.yml @@ -673,8 +673,6 @@ _tkinter: sources: - _tkinter.c - tkappinit.c - # TODO consider adding WITH_TIX, as Modules/Setup seems to recommend it. This also - # initializes tix at init time, which seems desirable. defines: - WITH_APPINIT includes-deps: diff --git a/cpython-unix/targets.yml b/cpython-unix/targets.yml index 33db80c8..d0ef4a78 100644 --- a/cpython-unix/targets.yml +++ b/cpython-unix/targets.yml @@ -195,7 +195,6 @@ aarch64-unknown-linux-gnu: - sqlite - tcl - tk - - tix - uuid - xorgproto - xz @@ -285,7 +284,6 @@ armv7-unknown-linux-gnueabi: - sqlite - tcl - tk - - tix - uuid - xorgproto - xz @@ -327,7 +325,6 @@ armv7-unknown-linux-gnueabihf: - sqlite - tcl - tk - - tix - uuid - xorgproto - xz @@ -369,7 +366,6 @@ mips-unknown-linux-gnu: - sqlite - tcl - tk - - tix - uuid - xorgproto - xz @@ -411,7 +407,6 @@ mipsel-unknown-linux-gnu: - sqlite - tcl - tk - - tix - uuid - xorgproto - xz @@ -453,7 +448,6 @@ ppc64le-unknown-linux-gnu: - sqlite - tcl - tk - - tix - uuid - xorgproto - xz @@ -495,7 +489,6 @@ riscv64-unknown-linux-gnu: - sqlite - tcl - tk - - tix - uuid - xorgproto - xz @@ -537,7 +530,6 @@ s390x-unknown-linux-gnu: - sqlite - tcl - tk - - tix - uuid - xorgproto - xz @@ -812,7 +804,6 @@ x86_64-unknown-linux-gnu: - sqlite - tcl - tk - - tix - uuid - xorgproto - xz @@ -860,7 +851,6 @@ x86_64_v2-unknown-linux-gnu: - sqlite - tcl - tk - - tix - uuid - xorgproto - xz @@ -908,7 +898,6 @@ x86_64_v3-unknown-linux-gnu: - sqlite - tcl - tk - - tix - uuid - xorgproto - xz @@ -956,7 +945,6 @@ x86_64_v4-unknown-linux-gnu: - sqlite - tcl - tk - - tix - uuid - xorgproto - xz @@ -1002,7 +990,6 @@ x86_64-unknown-linux-musl: - sqlite - tcl - tk - - tix - uuid - xorgproto - xz @@ -1048,7 +1035,6 @@ x86_64_v2-unknown-linux-musl: - sqlite - tcl - tk - - tix - uuid - xorgproto - xz @@ -1094,7 +1080,6 @@ x86_64_v3-unknown-linux-musl: - sqlite - tcl - tk - - tix - uuid - xorgproto - xz @@ -1140,7 +1125,6 @@ x86_64_v4-unknown-linux-musl: - sqlite - tcl - tk - - tix - uuid - xorgproto - xz diff --git a/docs/distributions.rst b/docs/distributions.rst index 3dd6125a..9e4cf830 100644 --- a/docs/distributions.rst +++ b/docs/distributions.rst @@ -388,8 +388,8 @@ license_path tcl_library_path Relative path to location of tcl library files. The path should be a directory tree containing tcl files to support the tkinter extension. - This will include a subset of the library files provided by the tcl, tk, - and tix packages. + This will include a subset of the library files provided by the tcl + and tk packages. This points to the root directory containing tcl resources. Actual tcl resources are in sub-directories underneath, as identified by diff --git a/docs/quirks.rst b/docs/quirks.rst index 8900cd6a..f24886a1 100644 --- a/docs/quirks.rst +++ b/docs/quirks.rst @@ -71,18 +71,21 @@ ncurses/libedit/readline are loaded. .. _quirk_macos_no_tix: -No tix on macOS -=============== +No tix on UNIX +============== + +Tix is an old widget library for Tcl/Tk. Python previously had a wrapper +for it in ``tkinter.tix``, but it was deprecated in Python 3.6 (the +recommendation is to use ``tkinter.ttk``) and removed in Python 3.13. -macOS distributions do not contain tix tcl support files. This means that -``tkinter.tix`` module functionality will likely break at run-time. The -module will import fine. But attempting to instantiate a ``tkinter.tix.Tk`` -instance or otherwise attempt to run tix tcl files will result in a run-time -error. +The macOS and Linux distributions from this project do not build and +ship Tix, even for Python versions 3.12 and below. -``tkinter.tix`` has been deprecated since Python 3.6 and the official Python -macOS installers do not ship the tix support files. So this project behaves -similarly to the official CPython distributions. +We had previously attempted to ship Tix support on Linux, but it was +broken and nobody reported an issue about it. The macOS distributions +from this project never shipped support for Tix. The official Python.org +macOS installers and Apple's build of Python do not ship support for +Tix, either, so this project behaves similarly to those distributions. .. _quirk_windows_no_pip: diff --git a/docs/status.rst b/docs/status.rst index ef172e04..cd50e078 100644 --- a/docs/status.rst +++ b/docs/status.rst @@ -285,7 +285,7 @@ test_spwd test_startfile object has no attribute 'startfile' test_tix - cannot run without OS X gui process + tix is not built by this project test_tk cannot run without OS X gui process test_ttk_guionly From 7d3476dc70f59555e57c23d1434e2971ac422e51 Mon Sep 17 00:00:00 2001 From: Geoffrey Thomas Date: Sat, 28 Jun 2025 22:36:11 -0400 Subject: [PATCH 3/8] Build libtcl, libtk, and _tkinter as shared objects --- cpython-unix/build-cpython.sh | 19 ++++++++++++++++++- cpython-unix/build-tcl.sh | 16 +++++++++------- cpython-unix/build-tk.sh | 24 +++++++++++++++--------- cpython-unix/extension-modules.yml | 24 ++++-------------------- pythonbuild/cpython.py | 13 ++++++++++++- src/validation.rs | 2 +- 6 files changed, 59 insertions(+), 39 deletions(-) diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index e28c70bc..5443bb66 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -44,7 +44,7 @@ sed "${sed_args[@]}" "s|/tools/host|${TOOLS_PATH}/host|g" ${TOOLS_PATH}/host/sha # We force linking of external static libraries by removing the shared # libraries. This is hacky. But we're building in a temporary container # and it gets the job done. -find ${TOOLS_PATH}/deps -name '*.so*' -exec rm {} \; +find ${TOOLS_PATH}/deps -name '*.so*' -a \! \( -name 'libtcl*.so*' -or -name 'libtk*.so*' \) -exec rm {} \; tar -xf Python-${PYTHON_VERSION}.tar.xz @@ -705,6 +705,8 @@ if [ "${PYBUILD_SHARED}" = "1" ]; then ${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION} # Python's build system doesn't make this file writable. + # TODO(geofft): @executable_path/ is a weird choice here, who is + # relying on it? Should probably be @loader_path. chmod 755 ${ROOT}/out/python/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME} install_name_tool \ -change /install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME} @executable_path/${LIBPYTHON_SHARED_LIBRARY_BASENAME} \ @@ -723,6 +725,13 @@ if [ "${PYBUILD_SHARED}" = "1" ]; then -change /install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME} @executable_path/../lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME} \ ${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX} fi + + # At the moment, python3 and libpython don't have shared-library + # dependencies, but at some point we will want to run this for + # them too. + for module in ${ROOT}/out/python/install/lib/python*/lib-dynload/*.so; do + install_name_tool -add_rpath @loader_path/../.. "$module" + done else # (not macos) LIBPYTHON_SHARED_LIBRARY_BASENAME=libpython${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX}.so.1.0 LIBPYTHON_SHARED_LIBRARY=${ROOT}/out/python/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME} @@ -1250,6 +1259,14 @@ if [ -d "${TOOLS_PATH}/deps/lib/tcl8" ]; then for source in ${TOOLS_PATH}/deps/lib/{itcl4.2.4,tcl8,tcl8.6,thread2.8.9,tk8.6}; do cp -av $source ${ROOT}/out/python/install/lib/ done + + ( + shopt -s nullglob + dylibs=(${TOOLS_PATH}/deps/lib/lib*.dylib ${TOOLS_PATH}/deps/lib/lib*.so) + if [ "${#dylibs[@]}" -gt 0 ]; then + cp -av "${dylibs[@]}" ${ROOT}/out/python/install/lib/ + fi + ) fi # Copy the terminfo database if present. diff --git a/cpython-unix/build-tcl.sh b/cpython-unix/build-tcl.sh index 43a4a6ad..af5c3062 100755 --- a/cpython-unix/build-tcl.sh +++ b/cpython-unix/build-tcl.sh @@ -20,9 +20,8 @@ if [ -n "${STATIC}" ]; then # `checking whether musl-clang accepts -g...` fails with a duplicate definition error TARGET_TRIPLE="$(echo "${TARGET_TRIPLE}" | sed -e 's/-unknown-linux-musl/-unknown-linux-gnu/g')" fi -fi -patch -p1 << 'EOF' + patch -p1 << 'EOF' diff --git a/unix/Makefile.in b/unix/Makefile.in --- a/unix/Makefile.in +++ b/unix/Makefile.in @@ -36,6 +35,7 @@ diff --git a/unix/Makefile.in b/unix/Makefile.in fi; \ fi; \ EOF +fi # Remove packages we don't care about and can pull in unwanted symbols. rm -rf pkgs/sqlite* pkgs/tdbc* @@ -48,12 +48,14 @@ CFLAGS="${CFLAGS}" CPPFLAGS="${CFLAGS}" LDFLAGS="${EXTRA_TARGET_LDFLAGS}" ./conf --build=${BUILD_TRIPLE} \ --host=${TARGET_TRIPLE} \ --prefix=/tools/deps \ - --enable-shared=no \ + --enable-shared"${STATIC:+=no}" \ --enable-threads -make -j ${NUM_CPUS} -make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out +make -j ${NUM_CPUS} DYLIB_INSTALL_DIR=@rpath +make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out DYLIB_INSTALL_DIR=@rpath make -j ${NUM_CPUS} install-private-headers DESTDIR=${ROOT}/out -# For some reason libtcl*.a have weird permissions. Fix that. -chmod 644 ${ROOT}/out/tools/deps/lib/libtcl*.a +if [ -n "${STATIC}" ]; then + # For some reason libtcl*.a have weird permissions. Fix that. + chmod 644 ${ROOT}/out/tools/deps/lib/libtcl*.a +fi diff --git a/cpython-unix/build-tk.sh b/cpython-unix/build-tk.sh index 2769a631..a6a5afb6 100755 --- a/cpython-unix/build-tk.sh +++ b/cpython-unix/build-tk.sh @@ -31,7 +31,7 @@ CFLAGS="${CFLAGS}" CPPFLAGS="${CFLAGS}" LDFLAGS="${LDFLAGS}" ./configure \ --host=${TARGET_TRIPLE} \ --prefix=/tools/deps \ --with-tcl=${TOOLS_PATH}/deps/lib \ - --enable-shared=no \ + --enable-shared"${STATIC:+=no}" \ --enable-threads \ ${EXTRA_CONFIGURE_FLAGS} @@ -41,19 +41,25 @@ if [[ "${PYBUILD_PLATFORM}" != macos* ]]; then sed -i 's/install-binaries: $(TK_STUB_LIB_FILE) $(TK_LIB_FILE) ${WISH_EXE}/install-binaries: $(TK_STUB_LIB_FILE) $(TK_LIB_FILE)/' Makefile fi -# For some reason musl isn't link libXau and libxcb. So we hack the Makefile -# to do what we want. -if [ "${CC}" = "musl-clang" ]; then - sed -i 's/-ldl -lpthread /-ldl -lpthread -lXau -lxcb/' tkConfig.sh - sed -i 's/-lpthread $(X11_LIB_SWITCHES) -ldl -lpthread/-lpthread $(X11_LIB_SWITCHES) -ldl -lpthread -lXau -lxcb/' Makefile +# We are statically linking libX11, and static libraries do not carry +# information about dependencies. pkg-config --static does, but Tcl/Tk's +# build system apparently is too old for that. So we need to manually +# inform the build process that libX11.a needs libxcb.a and libXau.a. +# Note that the order is significant, for static libraries: X11 requires +# xcb, which requires Xau. +MAKE_VARS=(DYLIB_INSTALL_DIR=@rpath) +if [[ "${PYBUILD_PLATFORM}" != macos* ]]; then + MAKE_VARS+=(X11_LIB_SWITCHES="-lX11 -lxcb -lXau") fi -make -j ${NUM_CPUS} +make -j ${NUM_CPUS} "${MAKE_VARS[@]}" touch wish -make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out +make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out "${MAKE_VARS[@]}" make -j ${NUM_CPUS} install-private-headers DESTDIR=${ROOT}/out # For some reason libtk*.a have weird permissions. Fix that. -chmod 644 /${ROOT}/out/tools/deps/lib/libtk*.a +if [ -n "${STATIC}" ]; then + chmod 644 /${ROOT}/out/tools/deps/lib/libtk*.a +fi rm ${ROOT}/out/tools/deps/bin/wish* diff --git a/cpython-unix/extension-modules.yml b/cpython-unix/extension-modules.yml index 8f74aa55..a3b750df 100644 --- a/cpython-unix/extension-modules.yml +++ b/cpython-unix/extension-modules.yml @@ -677,39 +677,23 @@ _tkinter: - WITH_APPINIT includes-deps: - include/X11 + build-mode: shared links: - tcl8.6 - tk8.6 - - # Without -ObjC, we get a crash: -[TKApplication tkProcessEvent:]: unrecognized selector sent to instance. - # See also https://core.tcl-lang.org/tk/tktview/85f316beb15108ac43b03fa6c8608e31f3ae5f92. - # This is apparently an issue with static linking Objective-C binaries. - linker-args: - - args: ["-ObjC"] - targets: - - .*-apple-darwin links-conditional: - name: X11 targets: - .*-unknown-linux-.* + build-mode: static - name: xcb targets: - .*-unknown-linux-.* + build-mode: static - name: Xau targets: - .*-unknown-linux-.* - # Many of these are dependencies of libtcl and libtk. - frameworks: - - AppKit - - ApplicationServices - - Carbon - - Cocoa - - CoreFoundation - - CoreServices - - CoreGraphics - - IOKit - - QuartzCore - - UniformTypeIdentifiers + build-mode: static _tokenize: minimum-python-version: "3.11" diff --git a/pythonbuild/cpython.py b/pythonbuild/cpython.py index c52f7ce6..bd7213b6 100644 --- a/pythonbuild/cpython.py +++ b/pythonbuild/cpython.py @@ -69,6 +69,7 @@ "properties": { "name": {"type": "string"}, "targets": {"type": "array", "items": {"type": "string"}}, + "build-mode": {"type": "string"}, }, "additionalProperties": False, }, @@ -535,7 +536,17 @@ def derive_setup_local( python_version, entry.get("maximum-python-version", "100.0") ) - if target_match and (python_min_match and python_max_match): + if build_mode := entry.get("build-mode"): + build_mode_match = section == build_mode + else: + build_mode_match = True + + if ( + target_match + and python_min_match + and python_max_match + and build_mode_match + ): if source := entry.get("source"): line += f" {source}" for source in entry.get("sources", []): diff --git a/src/validation.rs b/src/validation.rs index b0ffb530..cdd9ec54 100644 --- a/src/validation.rs +++ b/src/validation.rs @@ -822,7 +822,7 @@ const GLOBAL_EXTENSIONS_WINDOWS_PRE_3_13: &[&str] = &["_msi"]; const GLOBAL_EXTENSIONS_WINDOWS_NO_STATIC: &[&str] = &["_testinternalcapi", "_tkinter"]; /// Extension modules that should be built as shared libraries. -const SHARED_LIBRARY_EXTENSIONS: &[&str] = &["_crypt"]; +const SHARED_LIBRARY_EXTENSIONS: &[&str] = &["_crypt", "_tkinter"]; const PYTHON_VERIFICATIONS: &str = include_str!("verify_distribution.py"); From 30627178460c604b2690fb6eddf8a1cad09d757f Mon Sep 17 00:00:00 2001 From: Geoffrey Thomas Date: Mon, 4 Aug 2025 08:43:08 -0400 Subject: [PATCH 4/8] Fix validation for Tcl/Tk --- Cargo.toml | 2 +- cpython-unix/build-tcl.sh | 6 +++ cpython-unix/build-tk.sh | 6 +++ src/validation.rs | 81 ++++++++++++++++++++++++++++++++++----- 4 files changed, 85 insertions(+), 10 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b74eeb6e..beca9e80 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "pythonbuild" version = "0.1.0" authors = ["Gregory Szorc "] -edition = "2021" +edition = "2024" [dependencies] anyhow = "1.0.80" diff --git a/cpython-unix/build-tcl.sh b/cpython-unix/build-tcl.sh index af5c3062..ffc10c19 100755 --- a/cpython-unix/build-tcl.sh +++ b/cpython-unix/build-tcl.sh @@ -7,6 +7,12 @@ set -ex ROOT=`pwd` +# Force linking to static libraries from our dependencies. +# TODO(geofft): This is copied from build-cpython.sh. Really this should +# be done at the end of the build of each dependency, rather than before +# the build of each consumer. +find ${TOOLS_PATH}/deps -name '*.so*' -exec rm {} \; + export PATH=${TOOLS_PATH}/${TOOLCHAIN}/bin:${TOOLS_PATH}/host/bin:$PATH export PKG_CONFIG_PATH=${TOOLS_PATH}/deps/share/pkgconfig:${TOOLS_PATH}/deps/lib/pkgconfig diff --git a/cpython-unix/build-tk.sh b/cpython-unix/build-tk.sh index a6a5afb6..d67e6e9d 100755 --- a/cpython-unix/build-tk.sh +++ b/cpython-unix/build-tk.sh @@ -7,6 +7,12 @@ set -ex ROOT=`pwd` +# Force linking to static libraries from our dependencies. +# TODO(geofft): This is copied from build-cpython.sh. Really this should +# be done at the end of the build of each dependency, rather than before +# the build of each consumer. +find ${TOOLS_PATH}/deps -name '*.so*' -exec rm {} \; + export PATH=${TOOLS_PATH}/deps/bin:${TOOLS_PATH}/${TOOLCHAIN}/bin:${TOOLS_PATH}/host/bin:$PATH export PKG_CONFIG_PATH=${TOOLS_PATH}/deps/share/pkgconfig:${TOOLS_PATH}/deps/lib/pkgconfig diff --git a/src/validation.rs b/src/validation.rs index cdd9ec54..3b1ef5c8 100644 --- a/src/validation.rs +++ b/src/validation.rs @@ -9,7 +9,7 @@ use { normalize_path::NormalizePath, object::{ elf::{ - FileHeader32, FileHeader64, ET_DYN, ET_EXEC, STB_GLOBAL, STB_WEAK, STV_DEFAULT, + FileHeader32, FileHeader64, ET_DYN, ET_EXEC, SHN_UNDEF, STB_GLOBAL, STB_WEAK, STV_DEFAULT, STV_HIDDEN, }, macho::{MachHeader32, MachHeader64, MH_OBJECT, MH_TWOLEVEL}, @@ -265,6 +265,25 @@ static ELF_ALLOWED_LIBRARIES_BY_TRIPLE: Lazy>> = + Lazy::new(|| { + [ + ( + // libcrypt is provided by the system, but only on older distros. + "_crypt", + vec!["libcrypt.so.1"], + ), + ( + // libtcl and libtk are shipped in our distribution. + "_tkinter", + vec!["libtcl8.6.so", "libtk8.6.so"], + ), + ] + .iter() + .cloned() + .collect() + }); + static DARWIN_ALLOWED_DYLIBS: Lazy> = Lazy::new(|| { [ MachOAllowedDylib { @@ -501,6 +520,29 @@ static IOS_ALLOWED_DYLIBS: Lazy> = Lazy::new(|| { .to_vec() }); +static ALLOWED_DYLIBS_BY_MODULE: Lazy>> = + Lazy::new(|| { + [( + // libtcl and libtk are shipped in our distribution. + "_tkinter", + vec![ + MachOAllowedDylib { + name: "@rpath/libtcl8.6.dylib".to_string(), + max_compatibility_version: "8.6.0".try_into().unwrap(), + required: true, + }, + MachOAllowedDylib { + name: "@rpath/libtk8.6.dylib".to_string(), + max_compatibility_version: "8.6.0".try_into().unwrap(), + required: true, + }, + ], + )] + .iter() + .cloned() + .collect() + }); + static PLATFORM_TAG_BY_TRIPLE: Lazy> = Lazy::new(|| { [ ("aarch64-apple-darwin", "macosx-11.0-arm64"), @@ -544,9 +586,12 @@ const ELF_BANNED_SYMBOLS: &[&str] = &[ /// We use this list to spot test behavior of symbols belonging to dependency packages. /// The list is obviously not complete. const DEPENDENCY_PACKAGE_SYMBOLS: &[&str] = &[ - // libX11 - "XClearWindow", - "XFlush", + /* TODO(geofft): Tk provides these as no-op stubs on macOS, make it + * stop doing that so we can reenable the check + * // libX11 + * "XClearWindow", + * "XFlush", + */ // OpenSSL "BIO_ADDR_new", "BN_new", @@ -591,6 +636,11 @@ const DEPENDENCY_PACKAGE_SYMBOLS: &[&str] = &[ // liblzma "lzma_index_init", "lzma_stream_encoder", +]; + +// TODO(geofft): Conditionally prohibit these exported symbols +// everywhere except libtcl and libtk. This should be a hashmap +const _DEPENDENCY_PACKAGE_SYMBOLS_BUNDLED: &[&str] = &[ // tcl "Tcl_Alloc", "Tcl_ChannelName", @@ -967,11 +1017,13 @@ fn validate_elf>( allowed_libraries.push("libc.so".to_string()); } - // Allow the _crypt extension module - and only it - to link against libcrypt, - // which is no longer universally present in Linux distros. + // Allow certain extension modules to link against shared libraries + // (either from the system or from our distribution). if let Some(filename) = path.file_name() { - if filename.to_string_lossy().starts_with("_crypt") { - allowed_libraries.push("libcrypt.so.1".to_string()); + if let Some((module, _)) = filename.to_string_lossy().split_once(".cpython-") { + if let Some(extra) = ELF_ALLOWED_LIBRARIES_BY_MODULE.get(module) { + allowed_libraries.extend(extra.iter().map(|x| x.to_string())); + } } } @@ -1109,6 +1161,7 @@ fn validate_elf>( // to prevent them from being exported. if DEPENDENCY_PACKAGE_SYMBOLS.contains(&name.as_ref()) && matches!(symbol.st_bind(), STB_GLOBAL | STB_WEAK) + && symbol.st_shndx(endian) != SHN_UNDEF && symbol.st_visibility() != STV_HIDDEN { context.errors.push(format!( @@ -1124,6 +1177,7 @@ fn validate_elf>( if filename.starts_with("libpython") && filename.ends_with(".so.1.0") && matches!(symbol.st_bind(), STB_GLOBAL | STB_WEAK) + && symbol.st_shndx(endian) != SHN_UNDEF && symbol.st_visibility() == STV_DEFAULT { context.libpython_exported_symbols.insert(name.to_string()); @@ -1225,7 +1279,16 @@ fn validate_macho>( dylib_names.push(lib.clone()); - let allowed = allowed_dylibs_for_triple(target_triple); + let mut allowed = allowed_dylibs_for_triple(target_triple); + // Allow certain extension modules to link against shared libraries + // (either from the system or from our distribution). + if let Some(filename) = path.file_name() { + if let Some((module, _)) = filename.to_string_lossy().split_once(".cpython-") { + if let Some(extra) = ALLOWED_DYLIBS_BY_MODULE.get(module) { + allowed.extend(extra.clone()); + } + } + } if let Some(entry) = allowed.iter().find(|l| l.name == lib) { let load_version = From 1a633b0a92e3be876522db62dc305ef037d8b309 Mon Sep 17 00:00:00 2001 From: Geoffrey Thomas Date: Mon, 4 Aug 2025 13:41:42 -0400 Subject: [PATCH 5/8] Add zlib as a build-dep of Tcl/Tk This allows the shared libraries to statically link libz.a. I think we were previously picking up the zlib headers from the OS, and we were previously building static libraries so that was okay - they got linked into CPython itself which had the zlib symbols. But now that we're building shared libraries, they're picking up a dependency on libz.so.1, presumably because we don't have a static zlib installed from the OS. --- cpython-unix/Makefile | 8 +++++++- cpython-unix/build-tcl.sh | 3 ++- cpython-unix/build.py | 6 +++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/cpython-unix/Makefile b/cpython-unix/Makefile index d3164616..7e564145 100644 --- a/cpython-unix/Makefile +++ b/cpython-unix/Makefile @@ -192,7 +192,13 @@ $(OUTDIR)/patchelf-$(PATCHELF_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPEN $(OUTDIR)/sqlite-$(SQLITE_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-sqlite.sh $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) sqlite -$(OUTDIR)/tcl-$(TCL_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-tcl.sh +TCL_DEPENDS = \ + $(PYTHON_DEP_DEPENDS) \ + $(HERE)/build-tcl.sh \ + $(OUTDIR)/zlib-$(ZLIB_VERSION)-$(PACKAGE_SUFFIX).tar \ + $(NULL) + +$(OUTDIR)/tcl-$(TCL_VERSION)-$(PACKAGE_SUFFIX).tar: $(TCL_DEPENDS) $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) tcl TK_DEPENDS = \ diff --git a/cpython-unix/build-tcl.sh b/cpython-unix/build-tcl.sh index ffc10c19..ab409cf2 100755 --- a/cpython-unix/build-tcl.sh +++ b/cpython-unix/build-tcl.sh @@ -49,8 +49,9 @@ rm -rf pkgs/sqlite* pkgs/tdbc* pushd unix CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC -I${TOOLS_PATH}/deps/include" +LDFLAGS="${EXTRA_TARGET_CFLAGS} -L${TOOLS_PATH}/deps/lib" -CFLAGS="${CFLAGS}" CPPFLAGS="${CFLAGS}" LDFLAGS="${EXTRA_TARGET_LDFLAGS}" ./configure \ +CFLAGS="${CFLAGS}" CPPFLAGS="${CFLAGS}" LDFLAGS="${LDFLAGS}" ./configure \ --build=${BUILD_TRIPLE} \ --host=${TARGET_TRIPLE} \ --prefix=/tools/deps \ diff --git a/cpython-unix/build.py b/cpython-unix/build.py index 25976ce5..ea44c294 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -1121,6 +1121,9 @@ def main(): "zstd", ): tools_path = "host" if action in ("m4", "patchelf") else "deps" + extra_archives = { + "tcl": {"zlib"}, + }.get(action) simple_build( settings, @@ -1131,6 +1134,7 @@ def main(): target_triple=target_triple, build_options=build_options, dest_archive=dest_archive, + extra_archives=extra_archives, tools_path=tools_path, ) @@ -1195,7 +1199,7 @@ def main(): ) elif action == "tk": - extra_archives = {"tcl"} + extra_archives = {"tcl", "zlib"} if not host_platform.startswith("macos_"): extra_archives |= { "libX11", From fcb743980f27680bd917f02084aac639db204461 Mon Sep 17 00:00:00 2001 From: Geoffrey Thomas Date: Mon, 4 Aug 2025 22:54:18 -0400 Subject: [PATCH 6/8] Suppress building wish on all platforms --- cpython-unix/build-tk.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cpython-unix/build-tk.sh b/cpython-unix/build-tk.sh index d67e6e9d..13fe2011 100755 --- a/cpython-unix/build-tk.sh +++ b/cpython-unix/build-tk.sh @@ -42,10 +42,13 @@ CFLAGS="${CFLAGS}" CPPFLAGS="${CFLAGS}" LDFLAGS="${LDFLAGS}" ./configure \ ${EXTRA_CONFIGURE_FLAGS} # Remove wish, since we don't need it. -if [[ "${PYBUILD_PLATFORM}" != macos* ]]; then - sed -i 's/all: binaries libraries doc/all: libraries/' Makefile - sed -i 's/install-binaries: $(TK_STUB_LIB_FILE) $(TK_LIB_FILE) ${WISH_EXE}/install-binaries: $(TK_STUB_LIB_FILE) $(TK_LIB_FILE)/' Makefile +if [[ "${PYBUILD_PLATFORM}" = macos* ]]; then + sed_args=(-i '' -e) +else + sed_args=(-i) fi +sed "${sed_args[@]}" 's/all: binaries libraries doc/all: libraries/' Makefile +sed "${sed_args[@]}" 's/install-binaries: $(TK_STUB_LIB_FILE) $(TK_LIB_FILE) ${WISH_EXE}/install-binaries: $(TK_STUB_LIB_FILE) $(TK_LIB_FILE)/' Makefile # We are statically linking libX11, and static libraries do not carry # information about dependencies. pkg-config --static does, but Tcl/Tk's From 359a430c96b96db0b6c79eb061502a2a435a98f9 Mon Sep 17 00:00:00 2001 From: Geoffrey Thomas Date: Wed, 6 Aug 2025 23:19:06 -0400 Subject: [PATCH 7/8] libX11: ./configure --disable-loadable-xcursor This should suppress a mismatch leading to an xcb assertion failure. --- cpython-unix/build-libX11.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cpython-unix/build-libX11.sh b/cpython-unix/build-libX11.sh index bb45028b..9be17eea 100755 --- a/cpython-unix/build-libX11.sh +++ b/cpython-unix/build-libX11.sh @@ -99,6 +99,11 @@ if [ -n "${CROSS_COMPILING}" ]; then esac fi +# Avoid dlopen("libXcursor.so.1") from the OS, which can go horribly wrong. We +# might not need to avoid this if we switch to shipping X11 as shared +# libraries, and ideally if we ship libXcursor ourselves. +EXTRA_FLAGS="${EXTRA_FLAGS} --disable-loadable-xcursor" + # CC_FOR_BUILD is here because configure doesn't look for `clang` when # cross-compiling. So we force it. CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC -I/tools/deps/include" \ From 41b6663c48ceaa4953273d42f254d77961187700 Mon Sep 17 00:00:00 2001 From: Geoffrey Thomas Date: Fri, 8 Aug 2025 11:43:20 -0400 Subject: [PATCH 8/8] Add --exclude-libs=ALL for libtcl and libtk on Linux In general, if libX.so depends on libY.a, we don't want libY's symbols dynamically exported by libX. We want libX's use of libY to be private. We use two mechanisms for making this happen. -fvisibility=hidden, which works on the build of libY to mark the symbols themselves as hidden in the object file, and -Wl,--exclude-libs=ALL, which works on the build of libX to instruct the linker not to re-export symbols from static libraries. We set -fvisibility=hidden on some platforms but not all. We use -Wl,--exclude-libs=ALL on the build of Python itself on all platforms, which up to this point built our only dynamic objects (bin/python and libpython). Now that libtcl and libtk are dynamic, we need to do the same thing. I'm not sure if -fvisibility=hidden actually does anything useful for us, given the use of -Wl,--exclude-libs=ALL, and I think the fact that it's only set on some platforms is an oversight and demonstrates that it's not needed. See #735, which removes it; the builds pass vaildation. --- cpython-unix/build-tcl.sh | 3 +++ cpython-unix/build-tk.sh | 1 + 2 files changed, 4 insertions(+) diff --git a/cpython-unix/build-tcl.sh b/cpython-unix/build-tcl.sh index ab409cf2..bfbc22fa 100755 --- a/cpython-unix/build-tcl.sh +++ b/cpython-unix/build-tcl.sh @@ -50,6 +50,9 @@ pushd unix CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC -I${TOOLS_PATH}/deps/include" LDFLAGS="${EXTRA_TARGET_CFLAGS} -L${TOOLS_PATH}/deps/lib" +if [[ "${PYBUILD_PLATFORM}" != macos* ]]; then + LDFLAGS="${LDFLAGS} -Wl,--exclude-libs,ALL" +fi CFLAGS="${CFLAGS}" CPPFLAGS="${CFLAGS}" LDFLAGS="${LDFLAGS}" ./configure \ --build=${BUILD_TRIPLE} \ diff --git a/cpython-unix/build-tk.sh b/cpython-unix/build-tk.sh index 13fe2011..bd936c8e 100755 --- a/cpython-unix/build-tk.sh +++ b/cpython-unix/build-tk.sh @@ -29,6 +29,7 @@ if [[ "${PYBUILD_PLATFORM}" = macos* ]]; then LDFLAGS="-L${TOOLS_PATH}/deps/lib" EXTRA_CONFIGURE_FLAGS="--enable-aqua=yes --without-x" else + LDFLAGS="${LDFLAGS} -Wl,--exclude-libs,ALL" EXTRA_CONFIGURE_FLAGS="--x-includes=${TOOLS_PATH}/deps/include --x-libraries=${TOOLS_PATH}/deps/lib" fi