From d43e5fe092e7f0d20ddeb2400751bc95fb11c86c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Br=C3=A9nainn=20Woodsend?= Date: Fri, 22 Nov 2024 19:53:41 +0000 Subject: [PATCH 01/10] ci(macos): Avoid linking against homebrew Homebrew binaries are always compiled for exactly the version they're installed on making them very un-portable. When a wheel is "repaired" by cibuildwheel, delocate-wheel pulls in _psycopg's dependencies (libpq.dylib, libssl.dylib and libcrypto.dylib) which, on a GitHub Actions macOS 14 runner, are provided by Homebrew and are therefore only macOS >= 14 compatible. The resultant wheel is therefore incompatible with all but the latest macOS versions. Build all dependencies from source so that we can set the deployment target to something sensible. Fixes #1753. --- .github/workflows/packages.yml | 15 ++-- scripts/build/build_libpq.sh | 127 ++++++++++++++++++++++++++++----- 2 files changed, 117 insertions(+), 25 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index b41c1bea9..ad1c69c57 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -5,6 +5,8 @@ on: env: PIP_BREAK_SYSTEM_PACKAGES: "1" + LIBPQ_VERSION: "16.0" + OPENSSL_VERSION: "1.1.1w" jobs: sdist: # {{{ @@ -59,10 +61,6 @@ jobs: linux: # {{{ if: true - env: - LIBPQ_VERSION: "16.0" - OPENSSL_VERSION: "1.1.1w" - strategy: fail-fast: false matrix: @@ -162,6 +160,12 @@ jobs: - name: Checkout repos uses: actions/checkout@v4 + - name: Build dependencies + run: ./scripts/build/build_libpq.sh + + - name: Show dependency tree + run: otool -L /tmp/libpq.build/lib/*.dylib + - name: Build wheels uses: pypa/cibuildwheel@v2.22.0 env: @@ -172,12 +176,11 @@ jobs: export PYTHONPATH={project} && python -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')" CIBW_ENVIRONMENT: >- - MACOSX_DEPLOYMENT_TARGET=${{ matrix.macver }}.0 PG_VERSION=16 PACKAGE_NAME=psycopg2-binary PSYCOPG2_TESTDB=postgres PSYCOPG2_TEST_FAST=1 - PATH="/usr/local/opt/postgresql@${PG_VERSION}/bin:$PATH" + PATH="/tmp/libpq.build/bin:$PATH" - name: Upload artifacts uses: actions/upload-artifact@v4 diff --git a/scripts/build/build_libpq.sh b/scripts/build/build_libpq.sh index 8a99fbe82..5281d629c 100755 --- a/scripts/build/build_libpq.sh +++ b/scripts/build/build_libpq.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Build a modern version of libpq and depending libs from source on Centos 5 +# Build a modern version of libpq and depending libs from source on Centos 5, Alpine or macOS set -euo pipefail set -x @@ -12,30 +12,69 @@ postgres_version="${LIBPQ_VERSION}" # last release: https://www.openssl.org/source/ openssl_version="${OPENSSL_VERSION}" +# last release: https://kerberos.org/dist/ +krb5_version="1.21.3" + +# last release: https://www.gnu.org/software/gettext/ +gettext_version="0.22.5" + # last release: https://openldap.org/software/download/ -ldap_version="2.6.3" +ldap_version="2.6.8" # last release: https://github.com/cyrusimap/cyrus-sasl/releases sasl_version="2.1.28" export LIBPQ_BUILD_PREFIX=${LIBPQ_BUILD_PREFIX:-/tmp/libpq.build} -if [[ -f "${LIBPQ_BUILD_PREFIX}/lib/libpq.so" ]]; then +case "$(uname)" in + Darwin) + ID=macos + library_suffix=dylib + ;; + + Linux) + source /etc/os-release + library_suffix=so + ;; + + *) + echo "$0: unexpected Operating system: '$(uname)'" >&2 + exit 1 + ;; +esac + +if [[ -f "${LIBPQ_BUILD_PREFIX}/lib/libpq.${library_suffix}" ]]; then echo "libpq already available: build skipped" >&2 exit 0 fi -source /etc/os-release - case "$ID" in centos) yum update -y yum install -y zlib-devel krb5-devel pam-devel + curl="$(which curl)" ;; alpine) apk upgrade apk add --no-cache zlib-dev krb5-dev linux-pam-dev openldap-dev openssl-dev + curl="$(which curl)" + ;; + + macos) + brew install automake m4 libtool + # If available, libpq seemingly insists on linking against homebrew's + # openssl no matter what so remove it. Since homebrew's curl depends on + # it, force use of system curl. + brew uninstall --force --ignore-dependencies openssl gettext + curl="/usr/bin/curl" + # The deployment target should be <= to that of the oldest supported Python version. + # e.g. https://www.python.org/downloads/release/python-380/ + if [ "$(uname -m)" == "x86_64" ]; then + export MACOSX_DEPLOYMENT_TARGET=10.9 + else + export MACOSX_DEPLOYMENT_TARGET=11.0 + fi ;; *) @@ -44,12 +83,12 @@ case "$ID" in ;; esac -if [ "$ID" == "centos" ]; then +if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then # Build openssl if needed openssl_tag="OpenSSL_${openssl_version//./_}" openssl_dir="openssl-${openssl_tag}" - if [ ! -d "${openssl_dir}" ]; then curl -sL \ + if [ ! -d "${openssl_dir}" ]; then "$curl" -sL \ https://github.com/openssl/openssl/archive/${openssl_tag}.tar.gz \ | tar xzf - @@ -70,7 +109,55 @@ if [ "$ID" == "centos" ]; then fi -if [ "$ID" == "centos" ]; then +if [ "$ID" == "macos" ]; then + + # Build kerberos if needed + krb5_dir="krb5-${krb5_version}/src" + if [ ! -d "${krb5_dir}" ]; then + "$curl" -sL \ + curl -sL "https://kerberos.org/dist/krb5/$(echo 1.21.3 | grep -oE '\d+\.\d+')/krb5-${krb5_version}.tar.gz" \ + | tar xzf - + + cd "${krb5_dir}" + + ./configure --prefix=${LIBPQ_BUILD_PREFIX} \ + CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib + make + else + cd "${krb5_dir}" + fi + + make install + cd ../.. + +fi + + +if [ "$ID" == "macos" ]; then + + # Build gettext if needed + gettext_dir="gettext-${gettext_version}" + if [ ! -d "${gettext_dir}" ]; then + "$curl" -sL \ + curl -sL "https://ftp.gnu.org/pub/gnu/gettext/gettext-${gettext_version}.tar.gz" \ + | tar xzf - + + cd "${gettext_dir}" + + ./configure --prefix=${LIBPQ_BUILD_PREFIX} --disable-java \ + CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib + make -C gettext-runtime all + else + cd "${gettext_dir}" + fi + + make -C gettext-runtime install + cd .. + +fi + + +if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then # Build libsasl2 if needed # The system package (cyrus-sasl-devel) causes an amazing error on i686: @@ -79,14 +166,14 @@ if [ "$ID" == "centos" ]; then sasl_tag="cyrus-sasl-${sasl_version}" sasl_dir="cyrus-sasl-${sasl_tag}" if [ ! -d "${sasl_dir}" ]; then - curl -sL \ + "$curl" -sL \ https://github.com/cyrusimap/cyrus-sasl/archive/${sasl_tag}.tar.gz \ | tar xzf - cd "${sasl_dir}" autoreconf -i - ./configure --prefix=${LIBPQ_BUILD_PREFIX} \ + ./configure --prefix=${LIBPQ_BUILD_PREFIX} --disable-macos-framework \ CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib make else @@ -102,13 +189,13 @@ if [ "$ID" == "centos" ]; then fi -if [ "$ID" == "centos" ]; then +if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then # Build openldap if needed ldap_tag="${ldap_version}" ldap_dir="openldap-${ldap_tag}" if [ ! -d "${ldap_dir}" ]; then - curl -sL \ + "$curl" -sL \ https://www.openldap.org/software/download/OpenLDAP/openldap-release/openldap-${ldap_tag}.tgz \ | tar xzf - @@ -129,7 +216,7 @@ if [ "$ID" == "centos" ]; then make -C libraries/liblber/ install make -C libraries/libldap/ install make -C include/ install - chmod +x ${LIBPQ_BUILD_PREFIX}/lib/{libldap,liblber}*.so* + chmod +x ${LIBPQ_BUILD_PREFIX}/lib/{libldap,liblber}*.${library_suffix}* cd .. fi @@ -139,17 +226,19 @@ fi postgres_tag="REL_${postgres_version//./_}" postgres_dir="postgres-${postgres_tag}" if [ ! -d "${postgres_dir}" ]; then - curl -sL \ + "$curl" -sL \ https://github.com/postgres/postgres/archive/${postgres_tag}.tar.gz \ | tar xzf - cd "${postgres_dir}" - # Match the default unix socket dir default with what defined on Ubuntu and - # Red Hat, which seems the most common location - sed -i 's|#define DEFAULT_PGSOCKET_DIR .*'\ + if [ "$ID" != "macos" ]; then + # Match the default unix socket dir default with what defined on Ubuntu and + # Red Hat, which seems the most common location + sed -i 's|#define DEFAULT_PGSOCKET_DIR .*'\ '|#define DEFAULT_PGSOCKET_DIR "/var/run/postgresql"|' \ - src/include/pg_config_manual.h + src/include/pg_config_manual.h + fi # Often needed, but currently set by the workflow # export LD_LIBRARY_PATH="${LIBPQ_BUILD_PREFIX}/lib" @@ -171,4 +260,4 @@ make -C src/bin/pg_config install make -C src/include install cd .. -find ${LIBPQ_BUILD_PREFIX} -name \*.so.\* -type f -exec strip --strip-unneeded {} \; +find ${LIBPQ_BUILD_PREFIX} -name \*.${library_suffix}.\* -type f -exec strip --strip-unneeded {} \; From 310bc75532fd842cb578a44b0deee755391e9af2 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 5 Jan 2025 02:37:42 +0100 Subject: [PATCH 02/10] ci(macos): move libpq build script to BEFORE_ALL build step This is is how it is organised in Linux. --- .github/workflows/packages.yml | 6 ------ scripts/build/wheel_macos_before_all.sh | 6 ++++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index ad1c69c57..7d241dd79 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -160,12 +160,6 @@ jobs: - name: Checkout repos uses: actions/checkout@v4 - - name: Build dependencies - run: ./scripts/build/build_libpq.sh - - - name: Show dependency tree - run: otool -L /tmp/libpq.build/lib/*.dylib - - name: Build wheels uses: pypa/cibuildwheel@v2.22.0 env: diff --git a/scripts/build/wheel_macos_before_all.sh b/scripts/build/wheel_macos_before_all.sh index 882b887fa..e5fdd1418 100755 --- a/scripts/build/wheel_macos_before_all.sh +++ b/scripts/build/wheel_macos_before_all.sh @@ -11,6 +11,12 @@ set -x dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" prjdir="$( cd "${dir}/../.." && pwd )" +# Build dependency libraries +"${prjdir}/scripts/build/build_libpq.sh" + +# Show dependency tree +# otool -L /tmp/libpq.build/lib/*.dylib + brew install gnu-sed postgresql@${PG_VERSION} brew link --overwrite postgresql@${PG_VERSION} From 65626ec56562d73aa0413b7e147991458a1c62af Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 5 Jan 2025 02:51:14 +0100 Subject: [PATCH 03/10] ci(macos): add libpq build caching --- .github/workflows/packages.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 7d241dd79..0ca5f7438 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -160,6 +160,12 @@ jobs: - name: Checkout repos uses: actions/checkout@v4 + - name: Cache libpq build + uses: actions/cache@v4 + with: + path: /tmp/libpq.build + key: libpq-${{ env.LIBPQ_VERSION }}-macos-${{ matrix.arch }} + - name: Build wheels uses: pypa/cibuildwheel@v2.22.0 env: From c8abc5ce61429da515750cc096a14cbbbf7a35bc Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 5 Jan 2025 03:19:29 +0100 Subject: [PATCH 04/10] ci(macos): no fast tests on macOS package building We don't run complete tests in CI, so let's not waste this chance. The overhead for complete tests is minimal compared to all the pipeline boilerplate. --- .github/workflows/packages.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 0ca5f7438..94db07be5 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -179,7 +179,6 @@ jobs: PG_VERSION=16 PACKAGE_NAME=psycopg2-binary PSYCOPG2_TESTDB=postgres - PSYCOPG2_TEST_FAST=1 PATH="/tmp/libpq.build/bin:$PATH" - name: Upload artifacts From 1eac4fd4da13eda54022e62216c837c9d563a18a Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 5 Jan 2025 03:48:41 +0100 Subject: [PATCH 05/10] test(macos): soften tests to account for macOS polling differences --- tests/test_async.py | 2 +- tests/test_green.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_async.py b/tests/test_async.py index dfc7b598b..975e7f4db 100755 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -504,7 +504,7 @@ def test_non_block_after_notification(self): raise Exception("Unexpected result from poll: %r", state) polls += 1 - self.assert_(polls >= 8, polls) + self.assert_(polls >= 5, polls) def test_poll_noop(self): self.conn.poll() diff --git a/tests/test_green.py b/tests/test_green.py index e558d6263..5e2ae5132 100755 --- a/tests/test_green.py +++ b/tests/test_green.py @@ -152,7 +152,7 @@ def wait(conn): """) polls = stub.polls.count(POLL_READ) - self.assert_(polls > 8, polls) + self.assert_(polls > 6, polls) class CallbackErrorTestCase(ConnectingTestCase): From d0bc154f311d2b790a9f809b8adb93def0165344 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Br=C3=A9nainn=20Woodsend?= Date: Sun, 5 Jan 2025 14:51:07 +0000 Subject: [PATCH 06/10] build(macos): Enable cross compiling libpq across macOS architectures The GitHub Actions runners look like they're only 1 year away from the last macOS x86_64 platform being removed. Get ahead of the game and build x86_64 on arm64. --- .github/workflows/packages.yml | 21 +++----------- scripts/build/build_libpq.sh | 52 +++++++++++++++++++++++----------- 2 files changed, 39 insertions(+), 34 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 94db07be5..c69489cd9 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -132,29 +132,15 @@ jobs: # }}} macos: # {{{ - runs-on: macos-${{ matrix.macver }} + runs-on: macos-latest if: true strategy: fail-fast: false matrix: # These archs require an Apple M1 runner: [arm64, universal2] - arch: [x86_64] + arch: [x86_64, arm64] pyver: [cp39, cp310, cp311, cp312, cp313] - macver: ["13"] - include: - - arch: arm64 - pyver: cp310 - macver: "14" - - arch: arm64 - pyver: cp311 - macver: "14" - - arch: arm64 - pyver: cp312 - macver: "14" - - arch: arm64 - pyver: cp313 - macver: "14" steps: - name: Checkout repos @@ -171,6 +157,7 @@ jobs: env: CIBW_BUILD: ${{matrix.pyver}}-macosx_${{matrix.arch}} CIBW_ARCHS_MACOS: ${{matrix.arch}} + MACOSX_ARCHITECTURE: ${{matrix.arch}} CIBW_BEFORE_ALL_MACOS: ./scripts/build/wheel_macos_before_all.sh CIBW_TEST_COMMAND: >- export PYTHONPATH={project} && @@ -184,7 +171,7 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: macos-${{matrix.pyver}}-macos-${{matrix.macver}}_${{matrix.arch}} + name: macos-${{matrix.pyver}}-macos-${{matrix.arch}} path: ./wheelhouse/*.whl # }}} diff --git a/scripts/build/build_libpq.sh b/scripts/build/build_libpq.sh index 5281d629c..83090a823 100755 --- a/scripts/build/build_libpq.sh +++ b/scripts/build/build_libpq.sh @@ -68,9 +68,12 @@ case "$ID" in # it, force use of system curl. brew uninstall --force --ignore-dependencies openssl gettext curl="/usr/bin/curl" - # The deployment target should be <= to that of the oldest supported Python version. + if [ -z "$MACOSX_ARCHITECTURE" ]; then + MACOSX_ARCHITECTURE="$(uname -m)" + fi + # Set the deployment target to be <= to that of the oldest supported Python version. # e.g. https://www.python.org/downloads/release/python-380/ - if [ "$(uname -m)" == "x86_64" ]; then + if [ "$MACOSX_ARCHITECTURE" == "x86_64" ]; then export MACOSX_DEPLOYMENT_TARGET=10.9 else export MACOSX_DEPLOYMENT_TARGET=11.0 @@ -83,6 +86,22 @@ case "$ID" in ;; esac + +if [ "$ID" == "macos" ]; then + make_configure_standard_flags=( \ + --prefix=${LIBPQ_BUILD_PREFIX} \ + "CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ -arch $MACOSX_ARCHITECTURE" \ + "LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib -arch $MACOSX_ARCHITECTURE" \ + ) +else + make_configure_standard_flags=( \ + --prefix=${LIBPQ_BUILD_PREFIX} \ + CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ \ + LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib \ + ) +fi + + if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then # Build openssl if needed @@ -94,8 +113,14 @@ if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then cd "${openssl_dir}" - ./config --prefix=${LIBPQ_BUILD_PREFIX} --openssldir=${LIBPQ_BUILD_PREFIX} \ - zlib -fPIC shared + options=(--prefix=${LIBPQ_BUILD_PREFIX} --openssldir=${LIBPQ_BUILD_PREFIX} \ + zlib -fPIC shared) + if [ -z "$MACOSX_ARCHITECTURE" ]; then + ./config $options + else + ./configure "darwin64-$MACOSX_ARCHITECTURE-cc" $options + fi + make depend make else @@ -119,9 +144,7 @@ if [ "$ID" == "macos" ]; then | tar xzf - cd "${krb5_dir}" - - ./configure --prefix=${LIBPQ_BUILD_PREFIX} \ - CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib + ./configure "${make_configure_standard_flags[@]}" make else cd "${krb5_dir}" @@ -143,9 +166,7 @@ if [ "$ID" == "macos" ]; then | tar xzf - cd "${gettext_dir}" - - ./configure --prefix=${LIBPQ_BUILD_PREFIX} --disable-java \ - CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib + ./configure --disable-java "${make_configure_standard_flags[@]}" make -C gettext-runtime all else cd "${gettext_dir}" @@ -173,8 +194,7 @@ if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then cd "${sasl_dir}" autoreconf -i - ./configure --prefix=${LIBPQ_BUILD_PREFIX} --disable-macos-framework \ - CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib + ./configure "${make_configure_standard_flags[@]}" --disable-macos-framework make else cd "${sasl_dir}" @@ -201,8 +221,7 @@ if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then cd "${ldap_dir}" - ./configure --prefix=${LIBPQ_BUILD_PREFIX} --enable-backends=no --enable-null \ - CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib + ./configure "${make_configure_standard_flags[@]}" --enable-backends=no --enable-null make depend make -C libraries/liblutil/ @@ -243,10 +262,9 @@ if [ ! -d "${postgres_dir}" ]; then # Often needed, but currently set by the workflow # export LD_LIBRARY_PATH="${LIBPQ_BUILD_PREFIX}/lib" - ./configure --prefix=${LIBPQ_BUILD_PREFIX} --sysconfdir=/etc/postgresql-common \ + ./configure "${make_configure_standard_flags[@]}" --sysconfdir=/etc/postgresql-common \ --with-gssapi --with-openssl --with-pam --with-ldap \ - --without-readline --without-icu \ - CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib + --without-readline --without-icu make -C src/interfaces/libpq make -C src/bin/pg_config make -C src/include From b943457896f0fdec3e8bd40a98baf8fd3fb87f7a Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 5 Jan 2025 20:20:06 +0100 Subject: [PATCH 07/10] test: drop brew curl to use the system one --- scripts/build/build_libpq.sh | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/scripts/build/build_libpq.sh b/scripts/build/build_libpq.sh index 83090a823..4136be304 100755 --- a/scripts/build/build_libpq.sh +++ b/scripts/build/build_libpq.sh @@ -52,13 +52,11 @@ case "$ID" in centos) yum update -y yum install -y zlib-devel krb5-devel pam-devel - curl="$(which curl)" ;; alpine) apk upgrade apk add --no-cache zlib-dev krb5-dev linux-pam-dev openldap-dev openssl-dev - curl="$(which curl)" ;; macos) @@ -66,8 +64,7 @@ case "$ID" in # If available, libpq seemingly insists on linking against homebrew's # openssl no matter what so remove it. Since homebrew's curl depends on # it, force use of system curl. - brew uninstall --force --ignore-dependencies openssl gettext - curl="/usr/bin/curl" + brew uninstall --force --ignore-dependencies openssl gettext curl if [ -z "$MACOSX_ARCHITECTURE" ]; then MACOSX_ARCHITECTURE="$(uname -m)" fi @@ -107,7 +104,8 @@ if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then # Build openssl if needed openssl_tag="OpenSSL_${openssl_version//./_}" openssl_dir="openssl-${openssl_tag}" - if [ ! -d "${openssl_dir}" ]; then "$curl" -sL \ + if [ ! -d "${openssl_dir}" ]; then + curl -sL \ https://github.com/openssl/openssl/archive/${openssl_tag}.tar.gz \ | tar xzf - @@ -139,8 +137,7 @@ if [ "$ID" == "macos" ]; then # Build kerberos if needed krb5_dir="krb5-${krb5_version}/src" if [ ! -d "${krb5_dir}" ]; then - "$curl" -sL \ - curl -sL "https://kerberos.org/dist/krb5/$(echo 1.21.3 | grep -oE '\d+\.\d+')/krb5-${krb5_version}.tar.gz" \ + curl -sL "https://kerberos.org/dist/krb5/${krb5_version%.*}/krb5-${krb5_version}.tar.gz" \ | tar xzf - cd "${krb5_dir}" @@ -161,8 +158,7 @@ if [ "$ID" == "macos" ]; then # Build gettext if needed gettext_dir="gettext-${gettext_version}" if [ ! -d "${gettext_dir}" ]; then - "$curl" -sL \ - curl -sL "https://ftp.gnu.org/pub/gnu/gettext/gettext-${gettext_version}.tar.gz" \ + curl -sL "https://ftp.gnu.org/pub/gnu/gettext/gettext-${gettext_version}.tar.gz" \ | tar xzf - cd "${gettext_dir}" @@ -187,7 +183,7 @@ if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then sasl_tag="cyrus-sasl-${sasl_version}" sasl_dir="cyrus-sasl-${sasl_tag}" if [ ! -d "${sasl_dir}" ]; then - "$curl" -sL \ + curl -sL \ https://github.com/cyrusimap/cyrus-sasl/archive/${sasl_tag}.tar.gz \ | tar xzf - @@ -215,7 +211,7 @@ if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then ldap_tag="${ldap_version}" ldap_dir="openldap-${ldap_tag}" if [ ! -d "${ldap_dir}" ]; then - "$curl" -sL \ + curl -sL \ https://www.openldap.org/software/download/OpenLDAP/openldap-release/openldap-${ldap_tag}.tgz \ | tar xzf - @@ -245,7 +241,7 @@ fi postgres_tag="REL_${postgres_version//./_}" postgres_dir="postgres-${postgres_tag}" if [ ! -d "${postgres_dir}" ]; then - "$curl" -sL \ + curl -sL \ https://github.com/postgres/postgres/archive/${postgres_tag}.tar.gz \ | tar xzf - From 5bfba4c96100cf354ee6f1bfe0bdc0956863fc5e Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 5 Jan 2025 20:49:22 +0100 Subject: [PATCH 08/10] refactor: use pushd/popd instead of cd --- scripts/build/build_libpq.sh | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/scripts/build/build_libpq.sh b/scripts/build/build_libpq.sh index 4136be304..a02cc90fb 100755 --- a/scripts/build/build_libpq.sh +++ b/scripts/build/build_libpq.sh @@ -109,7 +109,7 @@ if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then https://github.com/openssl/openssl/archive/${openssl_tag}.tar.gz \ | tar xzf - - cd "${openssl_dir}" + pushd "${openssl_dir}" options=(--prefix=${LIBPQ_BUILD_PREFIX} --openssldir=${LIBPQ_BUILD_PREFIX} \ zlib -fPIC shared) @@ -122,12 +122,12 @@ if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then make depend make else - cd "${openssl_dir}" + pushd "${openssl_dir}" fi # Install openssl make install_sw - cd .. + popd fi @@ -140,15 +140,15 @@ if [ "$ID" == "macos" ]; then curl -sL "https://kerberos.org/dist/krb5/${krb5_version%.*}/krb5-${krb5_version}.tar.gz" \ | tar xzf - - cd "${krb5_dir}" + pushd "${krb5_dir}" ./configure "${make_configure_standard_flags[@]}" make else - cd "${krb5_dir}" + pushd "${krb5_dir}" fi make install - cd ../.. + popd fi @@ -161,15 +161,15 @@ if [ "$ID" == "macos" ]; then curl -sL "https://ftp.gnu.org/pub/gnu/gettext/gettext-${gettext_version}.tar.gz" \ | tar xzf - - cd "${gettext_dir}" + pushd "${gettext_dir}" ./configure --disable-java "${make_configure_standard_flags[@]}" make -C gettext-runtime all else - cd "${gettext_dir}" + pushd "${gettext_dir}" fi make -C gettext-runtime install - cd .. + popd fi @@ -187,20 +187,20 @@ if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then https://github.com/cyrusimap/cyrus-sasl/archive/${sasl_tag}.tar.gz \ | tar xzf - - cd "${sasl_dir}" + pushd "${sasl_dir}" autoreconf -i ./configure "${make_configure_standard_flags[@]}" --disable-macos-framework make else - cd "${sasl_dir}" + pushd "${sasl_dir}" fi # Install libsasl2 # requires missing nroff to build touch saslauthd/saslauthd.8 make install - cd .. + popd fi @@ -215,7 +215,7 @@ if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then https://www.openldap.org/software/download/OpenLDAP/openldap-release/openldap-${ldap_tag}.tgz \ | tar xzf - - cd "${ldap_dir}" + pushd "${ldap_dir}" ./configure "${make_configure_standard_flags[@]}" --enable-backends=no --enable-null @@ -224,7 +224,7 @@ if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then make -C libraries/liblber/ make -C libraries/libldap/ else - cd "${ldap_dir}" + pushd "${ldap_dir}" fi # Install openldap @@ -232,7 +232,7 @@ if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then make -C libraries/libldap/ install make -C include/ install chmod +x ${LIBPQ_BUILD_PREFIX}/lib/{libldap,liblber}*.${library_suffix}* - cd .. + popd fi @@ -245,7 +245,7 @@ if [ ! -d "${postgres_dir}" ]; then https://github.com/postgres/postgres/archive/${postgres_tag}.tar.gz \ | tar xzf - - cd "${postgres_dir}" + pushd "${postgres_dir}" if [ "$ID" != "macos" ]; then # Match the default unix socket dir default with what defined on Ubuntu and @@ -265,13 +265,13 @@ if [ ! -d "${postgres_dir}" ]; then make -C src/bin/pg_config make -C src/include else - cd "${postgres_dir}" + pushd "${postgres_dir}" fi # Install libpq make -C src/interfaces/libpq install make -C src/bin/pg_config install make -C src/include install -cd .. +popd find ${LIBPQ_BUILD_PREFIX} -name \*.${library_suffix}.\* -type f -exec strip --strip-unneeded {} \; From cee23d83e09614c73cf35b83164e49603c2b546f Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 5 Jan 2025 21:18:02 +0100 Subject: [PATCH 09/10] chore(macos): drop unneeded gettext from libpq building --- scripts/build/build_libpq.sh | 24 ------------------------ scripts/build/wheel_macos_before_all.sh | 2 +- 2 files changed, 1 insertion(+), 25 deletions(-) diff --git a/scripts/build/build_libpq.sh b/scripts/build/build_libpq.sh index a02cc90fb..0baf37bed 100755 --- a/scripts/build/build_libpq.sh +++ b/scripts/build/build_libpq.sh @@ -15,9 +15,6 @@ openssl_version="${OPENSSL_VERSION}" # last release: https://kerberos.org/dist/ krb5_version="1.21.3" -# last release: https://www.gnu.org/software/gettext/ -gettext_version="0.22.5" - # last release: https://openldap.org/software/download/ ldap_version="2.6.8" @@ -153,27 +150,6 @@ if [ "$ID" == "macos" ]; then fi -if [ "$ID" == "macos" ]; then - - # Build gettext if needed - gettext_dir="gettext-${gettext_version}" - if [ ! -d "${gettext_dir}" ]; then - curl -sL "https://ftp.gnu.org/pub/gnu/gettext/gettext-${gettext_version}.tar.gz" \ - | tar xzf - - - pushd "${gettext_dir}" - ./configure --disable-java "${make_configure_standard_flags[@]}" - make -C gettext-runtime all - else - pushd "${gettext_dir}" - fi - - make -C gettext-runtime install - popd - -fi - - if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then # Build libsasl2 if needed diff --git a/scripts/build/wheel_macos_before_all.sh b/scripts/build/wheel_macos_before_all.sh index e5fdd1418..36e1c0b81 100755 --- a/scripts/build/wheel_macos_before_all.sh +++ b/scripts/build/wheel_macos_before_all.sh @@ -15,7 +15,7 @@ prjdir="$( cd "${dir}/../.." && pwd )" "${prjdir}/scripts/build/build_libpq.sh" # Show dependency tree -# otool -L /tmp/libpq.build/lib/*.dylib +otool -L /tmp/libpq.build/lib/*.dylib brew install gnu-sed postgresql@${PG_VERSION} brew link --overwrite postgresql@${PG_VERSION} From 6cd0fbdc492c60adf79ce4518e6b854f7facffb1 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 5 Jan 2025 21:51:57 +0100 Subject: [PATCH 10/10] fix(macos): don't crash on undefined variable --- scripts/build/build_libpq.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build/build_libpq.sh b/scripts/build/build_libpq.sh index 0baf37bed..107460f84 100755 --- a/scripts/build/build_libpq.sh +++ b/scripts/build/build_libpq.sh @@ -62,7 +62,7 @@ case "$ID" in # openssl no matter what so remove it. Since homebrew's curl depends on # it, force use of system curl. brew uninstall --force --ignore-dependencies openssl gettext curl - if [ -z "$MACOSX_ARCHITECTURE" ]; then + if [ -z "${MACOSX_ARCHITECTURE:-}" ]; then MACOSX_ARCHITECTURE="$(uname -m)" fi # Set the deployment target to be <= to that of the oldest supported Python version. @@ -110,7 +110,7 @@ if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then options=(--prefix=${LIBPQ_BUILD_PREFIX} --openssldir=${LIBPQ_BUILD_PREFIX} \ zlib -fPIC shared) - if [ -z "$MACOSX_ARCHITECTURE" ]; then + if [ -z "${MACOSX_ARCHITECTURE:-}" ]; then ./config $options else ./configure "darwin64-$MACOSX_ARCHITECTURE-cc" $options