From 26ea51e9a506c9cf975b4e19d7419d44d5b3bfa5 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sat, 25 Dec 2021 19:40:53 +0100 Subject: [PATCH 1/6] chore: update cibuildwheel This commit updates the build workflow to use the latest cibuildwheel as a GitHub Action. cibuildwheel configuration is now in its own file (as there's no `pyproject.toml` yet) Signed-off-by: mayeut --- .github/workflows/build.yml | 40 ++++++++++--------------------------- .gitignore | 1 + cibuildwheel.toml | 13 ++++++++++++ 3 files changed, 24 insertions(+), 30 deletions(-) create mode 100644 cibuildwheel.toml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6b600bf03..2dcc45c27 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,25 +20,15 @@ on: [push, pull_request] name: build jobs: - # Linux + macOS + Python 3 + # Linux + macOS + Python 3.6+ linux-macos-py3: - name: ${{ matrix.os }}-py3 + name: ${{ matrix.os }}-py36-plus runs-on: ${{ matrix.os }} timeout-minutes: 20 strategy: fail-fast: false matrix: - # os: [ubuntu-latest, macos-latest, windows-latest] - os: [ubuntu-latest, macos-10.15] - include: - - {name: Linux, python: '3.9', os: ubuntu-latest} - env: - CIBW_TEST_COMMAND: - PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_DEBUG=1 python {project}/psutil/tests/runner.py && - PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_DEBUG=1 python {project}/psutil/tests/test_memleaks.py - CIBW_TEST_EXTRAS: test - CIBW_BUILD: 'cp36-* cp37-* cp38-* cp39-* cp310-*' - CIBW_SKIP: '*-musllinux_*' + os: [ubuntu-20.04, macos-10.15] steps: - name: Cancel previous runs @@ -53,16 +43,10 @@ jobs: cache: pip cache-dependency-path: .github/workflows/build.yml - - name: Install cibuildwheel - run: pip install cibuildwheel - - # - name: (Windows) install Visual C++ for Python 2.7 - # if: matrix.os == 'windows-latest' - # run: | - # choco install vcpython27 -f -y - - name: Run tests - run: cibuildwheel . + uses: pypa/cibuildwheel@v2.10.1 + with: + config-file: "./cibuildwheel.toml" - name: Create wheels uses: actions/upload-artifact@v3 @@ -78,25 +62,21 @@ jobs: mv dist/psutil*.tar.gz wheelhouse/ python scripts/internal/print_hashes.py wheelhouse/ - # Linux + macOS + Python 2 + # Linux + macOS + Python 2.7 & 3.5 linux-macos-py2: - name: ${{ matrix.os }}-py2 + name: ${{ matrix.os }}-py27-py35 runs-on: ${{ matrix.os }} timeout-minutes: 20 strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-10.15] - include: - - {name: Linux, python: '3.9', os: ubuntu-latest} + os: [ubuntu-20.04, macos-10.15] env: - CIBW_ARCHS_LINUX: 'x86_64 i686' CIBW_TEST_COMMAND: PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_DEBUG=1 python {project}/psutil/tests/runner.py && PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_DEBUG=1 python {project}/psutil/tests/test_memleaks.py CIBW_TEST_EXTRAS: test - CIBW_BUILD: 'cp27-*' - CIBW_SKIP: '*-musllinux_*' + CIBW_BUILD: 'cp27-* cp35-*' steps: - name: Cancel previous runs diff --git a/.gitignore b/.gitignore index 3d22b0b35..ddafc64c6 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ syntax: glob .tox/ build/ dist/ +wheelhouse/ diff --git a/cibuildwheel.toml b/cibuildwheel.toml new file mode 100644 index 000000000..9cf2c4d90 --- /dev/null +++ b/cibuildwheel.toml @@ -0,0 +1,13 @@ +[tool.cibuildwheel] +skip = ["pp*", "*-musllinux*"] +test-extras = "test" +test-command = [ + "PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_DEBUG=1 python {project}/psutil/tests/runner.py", + "PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_DEBUG=1 python {project}/psutil/tests/test_memleaks.py" +] + +[tool.cibuildwheel.macos] +archs = ["x86_64", "universal2"] + +[tool.cibuildwheel.macos.environment] +MACOSX_DEPLOYMENT_TARGET = "10.9" From 91e5262fc0f1d9b57d96904333dcfe4d655069fe Mon Sep 17 00:00:00 2001 From: mayeut Date: Fri, 27 May 2022 20:03:42 +0200 Subject: [PATCH 2/6] chore: use macos-12 runner for FreeBSD tests Signed-off-by: mayeut --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2dcc45c27..9e82a03fc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -112,7 +112,7 @@ jobs: python scripts/internal/print_hashes.py wheelhouse/ freebsd: - runs-on: macos-10.15 + runs-on: macos-12 steps: - name: Cancel previous runs uses: styfle/cancel-workflow-action@0.6.0 From 1d656a7438a7cda502558b7f121414c881220bf7 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sat, 21 May 2022 12:02:39 +0200 Subject: [PATCH 3/6] chore: use `macos-11` runners `macos-10.15` runner is deprecated. Signed-off-by: mayeut --- .github/workflows/build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9e82a03fc..445640be9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,7 +28,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-20.04, macos-10.15] + os: [ubuntu-20.04, macos-11] steps: - name: Cancel previous runs @@ -70,13 +70,14 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-20.04, macos-10.15] + os: [ubuntu-20.04, macos-11] env: CIBW_TEST_COMMAND: PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_DEBUG=1 python {project}/psutil/tests/runner.py && PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_DEBUG=1 python {project}/psutil/tests/test_memleaks.py CIBW_TEST_EXTRAS: test CIBW_BUILD: 'cp27-* cp35-*' + CIBW_SKIP: 'cp35-macosx_x86_64' steps: - name: Cancel previous runs From 581d2c7d7d602c55ac763a22dc881abf150bcde2 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sat, 21 May 2022 13:03:16 +0200 Subject: [PATCH 4/6] chore: skip test_weird_environ on macOS 11+ Signed-off-by: mayeut --- psutil/tests/__init__.py | 8 +++++++- psutil/tests/test_process.py | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 6ddafc972..62ae3462f 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -18,6 +18,7 @@ import gc import inspect import os +import platform import random import re import select @@ -84,7 +85,7 @@ "HAS_CPU_AFFINITY", "HAS_CPU_FREQ", "HAS_ENVIRON", "HAS_PROC_IO_COUNTERS", "HAS_IONICE", "HAS_MEMORY_MAPS", "HAS_PROC_CPU_NUM", "HAS_RLIMIT", "HAS_SENSORS_BATTERY", "HAS_BATTERY", "HAS_SENSORS_FANS", - "HAS_SENSORS_TEMPERATURES", "HAS_MEMORY_FULL_INFO", + "HAS_SENSORS_TEMPERATURES", "HAS_MEMORY_FULL_INFO", "MACOS_11PLUS", # subprocesses 'pyrun', 'terminate', 'reap_children', 'spawn_testproc', 'spawn_zombie', 'spawn_children_pair', @@ -127,6 +128,11 @@ CI_TESTING = APPVEYOR or GITHUB_ACTIONS # are we a 64 bit process? IS_64BIT = sys.maxsize > 2 ** 32 +if MACOS: + _macos_version = platform.mac_ver()[0] + MACOS_11PLUS = tuple(map(int, _macos_version.split(".")[:2])) > (10, 15) +else: + MACOS_11PLUS = False # --- configurable defaults diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index b2c3b2c76..390fe42c1 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -49,6 +49,7 @@ from psutil.tests import HAS_PROC_IO_COUNTERS from psutil.tests import HAS_RLIMIT from psutil.tests import HAS_THREADS +from psutil.tests import MACOS_11PLUS from psutil.tests import PYPY from psutil.tests import PYTHON_EXE from psutil.tests import PsutilTestCase @@ -1426,6 +1427,7 @@ def clean_dict(d): @unittest.skipIf(not HAS_ENVIRON, "not supported") @unittest.skipIf(not POSIX, "POSIX only") + @unittest.skipIf(MACOS_11PLUS, "macOS 11+ not supported, issue #2084") def test_weird_environ(self): # environment variables can contain values without an equals sign code = textwrap.dedent(""" From 097e654a6a24086b248d1d75d958020e6d9d8a7a Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 1 May 2022 11:03:01 +0200 Subject: [PATCH 5/6] feature: use ABI3 for cp36+ Signed-off-by: mayeut --- .ci/appveyor/install.ps1 | 85 ------------------------------ .github/workflows/build.yml | 49 +++++++++++++++-- appveyor.yml | 45 ---------------- cibuildwheel.toml | 15 ++++++ psutil/_psutil_linux.c | 15 ++---- psutil/arch/windows/process_info.c | 2 +- setup.py | 51 ++++++++++++++---- 7 files changed, 108 insertions(+), 154 deletions(-) delete mode 100644 .ci/appveyor/install.ps1 diff --git a/.ci/appveyor/install.ps1 b/.ci/appveyor/install.ps1 deleted file mode 100644 index 3f0562825..000000000 --- a/.ci/appveyor/install.ps1 +++ /dev/null @@ -1,85 +0,0 @@ -# Sample script to install Python and pip under Windows -# Authors: Olivier Grisel and Kyle Kastner -# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ - -$BASE_URL = "https://www.python.org/ftp/python/" -$GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" -$GET_PIP_PATH = "C:\get-pip.py" - - -function DownloadPython ($python_version, $platform_suffix) { - $webclient = New-Object System.Net.WebClient - $filename = "python-" + $python_version + $platform_suffix + ".msi" - $url = $BASE_URL + $python_version + "/" + $filename - - $basedir = $pwd.Path + "\" - $filepath = $basedir + $filename - if (Test-Path $filename) { - Write-Host "Reusing" $filepath - return $filepath - } - - # Download and retry up to 5 times in case of network transient errors. - Write-Host "Downloading" $filename "from" $url - $retry_attempts = 3 - for($i=0; $i -lt $retry_attempts; $i++){ - try { - $webclient.DownloadFile($url, $filepath) - break - } - Catch [Exception]{ - Start-Sleep 1 - } - } - Write-Host "File saved at" $filepath - return $filepath -} - - -function InstallPython ($python_version, $architecture, $python_home) { - Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home - if (Test-Path $python_home) { - Write-Host $python_home "already exists, skipping." - return $false - } - if ($architecture -eq "32") { - $platform_suffix = "" - } else { - $platform_suffix = ".amd64" - } - $filepath = DownloadPython $python_version $platform_suffix - Write-Host "Installing" $filepath "to" $python_home - $args = "/qn /i $filepath TARGETDIR=$python_home" - Write-Host "msiexec.exe" $args - Start-Process -FilePath "msiexec.exe" -ArgumentList $args -Wait -Passthru - Write-Host "Python $python_version ($architecture) installation complete" - return $true -} - - -function InstallPip ($python_home) { - $pip_path = $python_home + "/Scripts/pip.exe" - $python_path = $python_home + "/python.exe" - if (-not(Test-Path $pip_path)) { - Write-Host "Installing pip..." - $webclient = New-Object System.Net.WebClient - $webclient.DownloadFile($GET_PIP_URL, $GET_PIP_PATH) - Write-Host "Executing:" $python_path $GET_PIP_PATH - Start-Process -FilePath "$python_path" -ArgumentList "$GET_PIP_PATH" -Wait -Passthru - } else { - Write-Host "pip already installed." - } -} - -function InstallPackage ($python_home, $pkg) { - $pip_path = $python_home + "/Scripts/pip.exe" - & $pip_path install $pkg -} - -function main () { - InstallPython $env:PYTHON_VERSION $env:PYTHON_ARCH $env:PYTHON - InstallPip $env:PYTHON - InstallPackage $env:PYTHON wheel -} - -main diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 445640be9..d89ced5a6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,15 +20,15 @@ on: [push, pull_request] name: build jobs: - # Linux + macOS + Python 3.6+ - linux-macos-py3: + # Linux + macOS + Windows Python 3.6+ + py3: name: ${{ matrix.os }}-py36-plus runs-on: ${{ matrix.os }} timeout-minutes: 20 strategy: fail-fast: false matrix: - os: [ubuntu-20.04, macos-11] + os: [ubuntu-20.04, macos-11, windows-2019] steps: - name: Cancel previous runs @@ -62,6 +62,49 @@ jobs: mv dist/psutil*.tar.gz wheelhouse/ python scripts/internal/print_hashes.py wheelhouse/ + # Windows cp37+ tests + # psutil tests do not like running from a virtualenv with python>=3.7 so + # not using cibuildwheel for those. run them "manually" with this job. + windows-py3-test: + name: windows-py3-test-${{ matrix.python }}-${{ matrix.architecture }} + needs: py3 + runs-on: windows-2019 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + python: ["3.7", "3.8", "3.9", "3.10", "3.11-dev"] + architecture: ["x86", "x64"] + + steps: + - name: Cancel previous runs + uses: styfle/cancel-workflow-action@0.9.1 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + with: + python-version: "${{ matrix.python }}" + architecture: "${{ matrix.architecture }}" + cache: pip + cache-dependency-path: .github/workflows/build.yml + - name: Download wheels + uses: actions/download-artifact@v3 + with: + name: wheels + path: wheelhouse + - name: Run tests + run: | + mkdir .tests + cd .tests + pip install $(find ../wheelhouse -name '*-cp36-abi3-${{ matrix.architecture == 'x86' && 'win32' || 'win_amd64'}}.whl')[test] + export PYTHONWARNINGS=always + export PYTHONUNBUFFERED=1 + export PSUTIL_DEBUG=1 + python ../psutil/tests/runner.py + python ../psutil/tests/test_memleaks.py + shell: bash + # Linux + macOS + Python 2.7 & 3.5 linux-macos-py2: name: ${{ matrix.os }}-py27-py35 diff --git a/appveyor.yml b/appveyor.yml index 4bbd51aeb..0752e610c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -24,62 +24,17 @@ environment: PYTHON_VERSION: "2.7.x" PYTHON_ARCH: "32" - - PYTHON: "C:\\Python36" - PYTHON_VERSION: "3.6.x" - PYTHON_ARCH: "32" - - - PYTHON: "C:\\Python37" - PYTHON_VERSION: "3.7.x" - PYTHON_ARCH: "32" - - - PYTHON: "C:\\Python38" - PYTHON_VERSION: "3.8.x" - PYTHON_ARCH: "32" - - - PYTHON: "C:\\Python39" - PYTHON_VERSION: "3.9.x" - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 - PYTHON_ARCH: "32" - - - PYTHON: "C:\\Python310" - PYTHON_VERSION: "3.10.x" - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 - PYTHON_ARCH: "32" - # 64 bits - PYTHON: "C:\\Python27-x64" PYTHON_VERSION: "2.7.x" PYTHON_ARCH: "64" - - PYTHON: "C:\\Python36-x64" - PYTHON_VERSION: "3.6.x" - PYTHON_ARCH: "64" - - - PYTHON: "C:\\Python37-x64" - PYTHON_VERSION: "3.7.x" - PYTHON_ARCH: "64" - - - PYTHON: "C:\\Python38-x64" - PYTHON_VERSION: "3.8.x" - PYTHON_ARCH: "64" - - - PYTHON: "C:\\Python39-x64" - PYTHON_VERSION: "3.9.x" - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 - PYTHON_ARCH: "64" - - - PYTHON: "C:\\Python310-x64" - PYTHON_VERSION: "3.10.x" - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 - PYTHON_ARCH: "64" init: - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%" install: - - "powershell .ci\\appveyor\\install.ps1" - # - ps: (new-object net.webclient).DownloadFile('https://raw.github.com/pypa/pip/master/contrib/get-pip.py', 'C:/get-pip.py') - "%WITH_COMPILER% %PYTHON%/python.exe -m pip --version" - "%WITH_COMPILER% %PYTHON%/python.exe -m pip install --upgrade --user setuptools pip" - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py setup-dev-env" diff --git a/cibuildwheel.toml b/cibuildwheel.toml index 9cf2c4d90..c280fdc18 100644 --- a/cibuildwheel.toml +++ b/cibuildwheel.toml @@ -11,3 +11,18 @@ archs = ["x86_64", "universal2"] [tool.cibuildwheel.macos.environment] MACOSX_DEPLOYMENT_TARGET = "10.9" + +[tool.cibuildwheel.windows] +# psutil tests do not like running from a virtualenv with python>=3.7 +# restrict build & tests to cp36 +# cp36-abi3 wheels will need to be tested outside cibuildwheel for python>=3.7 +build = "cp36-*" +test-command = [ + "python {project}/psutil/tests/runner.py", + "python {project}/psutil/tests/test_memleaks.py" +] + +[tool.cibuildwheel.windows.environment] +PYTHONWARNINGS = "always" +PYTHONUNBUFFERED = "1" +PSUTIL_DEBUG = "1" diff --git a/psutil/_psutil_linux.c b/psutil/_psutil_linux.c index 64cdf0b55..7ef55189c 100644 --- a/psutil/_psutil_linux.c +++ b/psutil/_psutil_linux.c @@ -298,29 +298,25 @@ psutil_proc_cpu_affinity_set(PyObject *self, PyObject *args) { pid_t pid; int i, seq_len; PyObject *py_cpu_set; - PyObject *py_cpu_seq = NULL; if (!PyArg_ParseTuple(args, _Py_PARSE_PID "O", &pid, &py_cpu_set)) return NULL; if (!PySequence_Check(py_cpu_set)) { - PyErr_Format(PyExc_TypeError, "sequence argument expected, got %s", - Py_TYPE(py_cpu_set)->tp_name); + PyErr_Format(PyExc_TypeError, "sequence argument expected, got %R", Py_TYPE(py_cpu_set)); goto error; } - py_cpu_seq = PySequence_Fast(py_cpu_set, "expected a sequence or integer"); - if (!py_cpu_seq) - goto error; - seq_len = PySequence_Fast_GET_SIZE(py_cpu_seq); + seq_len = PySequence_Size(py_cpu_set); CPU_ZERO(&cpu_set); for (i = 0; i < seq_len; i++) { - PyObject *item = PySequence_Fast_GET_ITEM(py_cpu_seq, i); + PyObject *item = PySequence_GetItem(py_cpu_set, i); #if PY_MAJOR_VERSION >= 3 long value = PyLong_AsLong(item); #else long value = PyInt_AsLong(item); #endif + Py_XDECREF(item); if ((value == -1) || PyErr_Occurred()) { if (!PyErr_Occurred()) PyErr_SetString(PyExc_ValueError, "invalid CPU value"); @@ -335,12 +331,9 @@ psutil_proc_cpu_affinity_set(PyObject *self, PyObject *args) { goto error; } - Py_DECREF(py_cpu_seq); Py_RETURN_NONE; error: - if (py_cpu_seq != NULL) - Py_DECREF(py_cpu_seq); return NULL; } #endif /* PSUTIL_HAVE_CPU_AFFINITY */ diff --git a/psutil/arch/windows/process_info.c b/psutil/arch/windows/process_info.c index d44c4eb75..1981d306a 100644 --- a/psutil/arch/windows/process_info.c +++ b/psutil/arch/windows/process_info.c @@ -578,7 +578,7 @@ psutil_get_cmdline(DWORD pid, int use_peb) { wcslen(szArglist[i])); if (py_unicode == NULL) goto out; - PyList_SET_ITEM(py_retlist, i, py_unicode); + PyList_SetItem(py_retlist, i, py_unicode); py_unicode = NULL; } ret = py_retlist; diff --git a/setup.py b/setup.py index 0f6716b3b..b5d4f1291 100755 --- a/setup.py +++ b/setup.py @@ -100,6 +100,14 @@ def get_version(): VERSION = get_version() macros.append(('PSUTIL_VERSION', int(VERSION.replace('.', '')))) +PY36_PLUS = sys.version_info[:2] >= (3, 6) +CP36_PLUS = PY36_PLUS and sys.implementation.name == "cpython" +if CP36_PLUS and (MACOS or LINUX or WINDOWS): + py_limited_api = {"py_limited_api": True} + macros.append(('Py_LIMITED_API', '0x03060000')) +else: + py_limited_api = {} + def get_description(): script = os.path.join(HERE, "scripts", "internal", "convert_readme.py") @@ -182,7 +190,8 @@ def get_winver(): "ws2_32", "PowrProf", "pdh", ], # extra_compile_args=["/W 4"], - # extra_link_args=["/DEBUG"] + # extra_link_args=["/DEBUG"], + **py_limited_api ) elif MACOS: @@ -197,7 +206,8 @@ def get_winver(): define_macros=macros, extra_link_args=[ '-framework', 'CoreFoundation', '-framework', 'IOKit' - ]) + ], + **py_limited_api) elif FREEBSD: macros.append(("PSUTIL_FREEBSD", 1)) @@ -214,7 +224,8 @@ def get_winver(): 'psutil/arch/freebsd/proc_socks.c', ], define_macros=macros, - libraries=["devstat"]) + libraries=["devstat"], + **py_limited_api) elif OPENBSD: macros.append(("PSUTIL_OPENBSD", 1)) @@ -228,7 +239,8 @@ def get_winver(): 'psutil/arch/openbsd/proc.c', ], define_macros=macros, - libraries=["kvm"]) + libraries=["kvm"], + **py_limited_api) elif NETBSD: macros.append(("PSUTIL_NETBSD", 1)) @@ -240,7 +252,8 @@ def get_winver(): 'psutil/arch/netbsd/socks.c', ], define_macros=macros, - libraries=["kvm"]) + libraries=["kvm"], + **py_limited_api) elif LINUX: def get_ethtool_macro(): @@ -276,7 +289,8 @@ def get_ethtool_macro(): ext = Extension( 'psutil._psutil_linux', sources=sources + ['psutil/_psutil_linux.c'], - define_macros=macros) + define_macros=macros, + **py_limited_api) elif SUNOS: macros.append(("PSUTIL_SUNOS", 1)) @@ -288,7 +302,8 @@ def get_ethtool_macro(): 'psutil/arch/solaris/environ.c' ], define_macros=macros, - libraries=['kstat', 'nsl', 'socket']) + libraries=['kstat', 'nsl', 'socket'], + **py_limited_api) elif AIX: macros.append(("PSUTIL_AIX", 1)) @@ -300,7 +315,8 @@ def get_ethtool_macro(): 'psutil/arch/aix/common.c', 'psutil/arch/aix/ifaddrs.c'], libraries=['perfstat'], - define_macros=macros) + define_macros=macros, + **py_limited_api) else: sys.exit('platform %s is not supported' % sys.platform) @@ -310,7 +326,8 @@ def get_ethtool_macro(): posix_extension = Extension( 'psutil._psutil_posix', define_macros=macros, - sources=sources) + sources=sources, + **py_limited_api) if SUNOS: def get_sunos_update(): # See https://serverfault.com/q/524883 @@ -341,11 +358,27 @@ def get_sunos_update(): else: extensions = [ext] +cmdclass = {} +if py_limited_api: + from wheel.bdist_wheel import bdist_wheel as _bdist_wheel + + class bdist_wheel_abi3(_bdist_wheel): + def finalize_options(self): + _bdist_wheel.finalize_options(self) + self.root_is_pure = False + + def get_tag(self): + python, abi, plat = _bdist_wheel.get_tag(self) + return python, "abi3", plat + + cmdclass["bdist_wheel"] = bdist_wheel_abi3 + def main(): kwargs = dict( name='psutil', version=VERSION, + cmdclass=cmdclass, description=__doc__ .replace('\n', ' ').strip() if __doc__ else '', long_description=get_description(), long_description_content_type='text/x-rst', From 0daf018ac430b153c6908e876f5e73d2369f9072 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 1 May 2022 17:23:13 +0200 Subject: [PATCH 6/6] feat: add linux aarch64 wheel Signed-off-by: mayeut --- .github/workflows/build.yml | 34 ++++++++++++++++++++++------------ psutil/tests/__init__.py | 2 ++ psutil/tests/test_contracts.py | 5 +++++ psutil/tests/test_linux.py | 5 ++++- psutil/tests/test_memleaks.py | 3 +++ psutil/tests/test_posix.py | 7 ++++++- psutil/tests/test_process.py | 18 +++++++++++++++++- psutil/tests/test_system.py | 3 +++ 8 files changed, 62 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d89ced5a6..7dd130aeb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,7 +3,7 @@ # # * Linux # * macOS -# * Windows (disabled) +# * Windows # * FreeBSD # # To skip certain builds see: @@ -20,15 +20,19 @@ on: [push, pull_request] name: build jobs: - # Linux + macOS + Windows Python 3.6+ - py3: - name: ${{ matrix.os }}-py36-plus + # Linux + macOS + Windows CPython 3.6+ + cp36: + name: cp36+, ${{ matrix.os }}, ${{ matrix.archs }} runs-on: ${{ matrix.os }} - timeout-minutes: 20 + timeout-minutes: 30 strategy: fail-fast: false matrix: - os: [ubuntu-20.04, macos-11, windows-2019] + include: + - {os: macos-11, archs: "x86_64 universal2"} + - {os: ubuntu-20.04, archs: "x86_64 i686"} + - {os: ubuntu-20.04, archs: "aarch64"} + - {os: windows-2019, archs: "AMD64 x86"} steps: - name: Cancel previous runs @@ -43,10 +47,16 @@ jobs: cache: pip cache-dependency-path: .github/workflows/build.yml + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + if: matrix.archs == 'aarch64' + - name: Run tests uses: pypa/cibuildwheel@v2.10.1 with: config-file: "./cibuildwheel.toml" + env: + CIBW_ARCHS: ${{ matrix.archs }} - name: Create wheels uses: actions/upload-artifact@v3 @@ -65,9 +75,9 @@ jobs: # Windows cp37+ tests # psutil tests do not like running from a virtualenv with python>=3.7 so # not using cibuildwheel for those. run them "manually" with this job. - windows-py3-test: - name: windows-py3-test-${{ matrix.python }}-${{ matrix.architecture }} - needs: py3 + windows-cp3-test: + name: windows tests ${{ matrix.python }} ${{ matrix.architecture }} + needs: cp36 runs-on: windows-2019 timeout-minutes: 20 strategy: @@ -105,9 +115,9 @@ jobs: python ../psutil/tests/test_memleaks.py shell: bash - # Linux + macOS + Python 2.7 & 3.5 - linux-macos-py2: - name: ${{ matrix.os }}-py27-py35 + # Linux + macOS + CPython 2.7 & 3.5 + linux-macos-cp27-cp35: + name: cp27/cp35, ${{ matrix.os }} runs-on: ${{ matrix.os }} timeout-minutes: 20 strategy: diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 62ae3462f..fca6caf18 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -86,6 +86,7 @@ "HAS_IONICE", "HAS_MEMORY_MAPS", "HAS_PROC_CPU_NUM", "HAS_RLIMIT", "HAS_SENSORS_BATTERY", "HAS_BATTERY", "HAS_SENSORS_FANS", "HAS_SENSORS_TEMPERATURES", "HAS_MEMORY_FULL_INFO", "MACOS_11PLUS", + "QEMU_USER", # subprocesses 'pyrun', 'terminate', 'reap_children', 'spawn_testproc', 'spawn_zombie', 'spawn_children_pair', @@ -126,6 +127,7 @@ APPVEYOR = 'APPVEYOR' in os.environ GITHUB_ACTIONS = 'GITHUB_ACTIONS' in os.environ or 'CIBUILDWHEEL' in os.environ CI_TESTING = APPVEYOR or GITHUB_ACTIONS +QEMU_USER = LINUX and GITHUB_ACTIONS and platform.machine() == 'aarch64' # are we a 64 bit process? IS_64BIT = sys.maxsize > 2 ** 32 if MACOS: diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index d376a3385..e3e149480 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -43,6 +43,7 @@ from psutil.tests import HAS_SENSORS_FANS from psutil.tests import HAS_SENSORS_TEMPERATURES from psutil.tests import PYPY +from psutil.tests import QEMU_USER from psutil.tests import SKIP_SYSCONS from psutil.tests import VALID_PROC_STATUSES from psutil.tests import PsutilTestCase @@ -278,6 +279,7 @@ def test_net_if_addrs(self): self.assertIsInstance(addr.netmask, (str, type(None))) self.assertIsInstance(addr.broadcast, (str, type(None))) + @unittest.skipIf(QEMU_USER, 'QEMU user not supported') def test_net_if_stats(self): # Duplicate of test_system.py. Keep it anyway. for ifname, info in psutil.net_if_stats().items(): @@ -425,6 +427,9 @@ def test_all(self): failures = [] for info in self.iter_proc_info(): for name, value in info.items(): + if QEMU_USER and name == 'status': + # @unittest.skipIf(QEMU_USER, "QEMU user not supported") + continue meth = getattr(self, name) try: meth(value, info) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 3e1afc4fb..a5a1c5913 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -36,6 +36,7 @@ from psutil.tests import HAS_GETLOADAVG from psutil.tests import HAS_RLIMIT from psutil.tests import PYPY +from psutil.tests import QEMU_USER from psutil.tests import TOLERANCE_DISK_USAGE from psutil.tests import TOLERANCE_SYS_MEM from psutil.tests import PsutilTestCase @@ -974,6 +975,7 @@ def test_ips(self): @unittest.skipIf(not LINUX, "LINUX only") +@unittest.skipIf(QEMU_USER, "QEMU user not supported") class TestSystemNetIfStats(PsutilTestCase): @unittest.skipIf(not which("ifconfig"), "ifconfig utility not available") @@ -1537,7 +1539,7 @@ def test_issue_687(self): with ThreadTask(): p = psutil.Process() threads = p.threads() - self.assertEqual(len(threads), 2) + self.assertEqual(len(threads), 3 if QEMU_USER else 2) tid = sorted(threads, key=lambda x: x.id)[1].id self.assertNotEqual(p.pid, tid) pt = psutil.Process(tid) @@ -2219,6 +2221,7 @@ def test_name(self): value = self.read_status_file("Name:") self.assertEqual(self.proc.name(), value) + @unittest.skipIf(QEMU_USER, "QEMU user not supported") def test_status(self): value = self.read_status_file("State:") value = value[value.find('(') + 1:value.rfind(')')] diff --git a/psutil/tests/test_memleaks.py b/psutil/tests/test_memleaks.py index e507e837c..9199a7f6b 100755 --- a/psutil/tests/test_memleaks.py +++ b/psutil/tests/test_memleaks.py @@ -43,6 +43,7 @@ from psutil.tests import HAS_SENSORS_BATTERY from psutil.tests import HAS_SENSORS_FANS from psutil.tests import HAS_SENSORS_TEMPERATURES +from psutil.tests import QEMU_USER from psutil.tests import TestMemoryLeak from psutil.tests import create_sockets from psutil.tests import get_testfn @@ -393,6 +394,7 @@ def test_disk_usage(self): times = FEW_TIMES if POSIX else self.times self.execute(lambda: psutil.disk_usage('.'), times=times) + @unittest.skipIf(QEMU_USER, "QEMU user not supported") def test_disk_partitions(self): self.execute(psutil.disk_partitions) @@ -428,6 +430,7 @@ def test_net_if_addrs(self): tolerance = 80 * 1024 if WINDOWS else self.tolerance self.execute(psutil.net_if_addrs, tolerance=tolerance) + @unittest.skipIf(QEMU_USER, "QEMU user not supported") def test_net_if_stats(self): self.execute(psutil.net_if_stats) diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index ebbf7a6e8..aafad2965 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -26,6 +26,7 @@ from psutil.tests import CI_TESTING from psutil.tests import HAS_NET_IO_COUNTERS from psutil.tests import PYTHON_EXE +from psutil.tests import QEMU_USER from psutil.tests import PsutilTestCase from psutil.tests import mock from psutil.tests import retry_on_failure @@ -107,7 +108,11 @@ def ps_name(pid): field = "command" if SUNOS: field = "comm" - return ps(field, pid).split()[0] + command = ps(field, pid).split() + if QEMU_USER: + assert "/bin/qemu-" in command[0] + return command[1] + return command[0] def ps_args(pid): diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 390fe42c1..12cf82189 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -52,6 +52,7 @@ from psutil.tests import MACOS_11PLUS from psutil.tests import PYPY from psutil.tests import PYTHON_EXE +from psutil.tests import QEMU_USER from psutil.tests import PsutilTestCase from psutil.tests import ThreadTask from psutil.tests import call_until @@ -243,6 +244,7 @@ def test_cpu_percent_numcpus_none(self): psutil.Process().cpu_percent() assert m.called + @unittest.skipIf(QEMU_USER, "QEMU user not supported") def test_cpu_times(self): times = psutil.Process().cpu_times() assert (times.user > 0.0) or (times.system > 0.0), times @@ -254,6 +256,7 @@ def test_cpu_times(self): for name in times._fields: time.strftime("%H:%M:%S", time.localtime(getattr(times, name))) + @unittest.skipIf(QEMU_USER, "QEMU user not supported") def test_cpu_times_2(self): user_time, kernel_time = psutil.Process().cpu_times()[:2] utime, ktime = os.times()[:2] @@ -616,6 +619,8 @@ def test_memory_maps(self): for nt in maps: if not nt.path.startswith('['): + if QEMU_USER and "/bin/qemu-" in nt.path: + continue assert os.path.isabs(nt.path), nt.path if POSIX: try: @@ -679,6 +684,7 @@ def test_is_running(self): assert not p.is_running() assert not p.is_running() + @unittest.skipIf(QEMU_USER, "QEMU user not supported") def test_exe(self): p = self.spawn_psproc() exe = p.exe() @@ -724,6 +730,9 @@ def test_cmdline(self): self.assertEqual(' '.join(p.cmdline()[1:]), ' '.join(cmdline[1:])) return + if QEMU_USER: + self.assertEqual(' '.join(p.cmdline()[2:]), ' '.join(cmdline)) + return self.assertEqual(' '.join(p.cmdline()), ' '.join(cmdline)) @unittest.skipIf(PYPY, "broken on PYPY") @@ -732,6 +741,9 @@ def test_long_cmdline(self): create_exe(testfn) cmdline = [testfn] + (["0123456789"] * 20) p = self.spawn_psproc(cmdline) + if QEMU_USER: + self.assertEqual(p.cmdline()[2:], cmdline) + return self.assertEqual(p.cmdline(), cmdline) def test_name(self): @@ -740,7 +752,8 @@ def test_name(self): pyexe = os.path.basename(os.path.realpath(sys.executable)).lower() assert pyexe.startswith(name), (pyexe, name) - @unittest.skipIf(PYPY, "unreliable on PYPY") + @unittest.skipIf(PYPY or QEMU_USER, "unreliable on PYPY") + @unittest.skipIf(QEMU_USER, "unreliable on QEMU user") def test_long_name(self): testfn = self.get_testfn(suffix="0123456789" * 2) create_exe(testfn) @@ -751,6 +764,7 @@ def test_long_name(self): @unittest.skipIf(SUNOS, "broken on SUNOS") @unittest.skipIf(AIX, "broken on AIX") @unittest.skipIf(PYPY, "broken on PYPY") + @unittest.skipIf(QEMU_USER, "broken on QEMU user") def test_prog_w_funky_name(self): # Test that name(), exe() and cmdline() correctly handle programs # with funky chars such as spaces and ")", see: @@ -848,6 +862,7 @@ def test_nice(self): except psutil.AccessDenied: pass + @unittest.skipIf(QEMU_USER, "QEMU user not supported") def test_status(self): p = psutil.Process() self.assertEqual(p.status(), psutil.STATUS_RUNNING) @@ -1073,6 +1088,7 @@ def test_parent_disappeared(self): side_effect=psutil.NoSuchProcess(0, 'foo')): self.assertIsNone(p.parent()) + @unittest.skipIf(QEMU_USER, "QEMU user not supported") @retry_on_failure() def test_parents(self): parent = psutil.Process() diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index d6b7a21a4..02dbcae01 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -45,6 +45,7 @@ from psutil.tests import HAS_SENSORS_TEMPERATURES from psutil.tests import IS_64BIT from psutil.tests import PYPY +from psutil.tests import QEMU_USER from psutil.tests import UNICODE_SUFFIX from psutil.tests import PsutilTestCase from psutil.tests import check_net_address @@ -723,6 +724,7 @@ def test_net_io_counters_no_nics(self): self.assertEqual(psutil.net_io_counters(pernic=True), {}) assert m.called + @unittest.skipIf(QEMU_USER, 'QEMU user not supported') def test_net_if_addrs(self): nics = psutil.net_if_addrs() assert nics, nics @@ -800,6 +802,7 @@ def test_net_if_addrs_mac_null_bytes(self): else: self.assertEqual(addr.address, '06-3d-29-00-00-00') + @unittest.skipIf(QEMU_USER, 'QEMU user not supported') def test_net_if_stats(self): nics = psutil.net_if_stats() assert nics, nics