From 6bdb735bbe97c94b9a8191259effca71c741c746 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Sun, 2 Mar 2025 18:25:22 +0800 Subject: [PATCH 1/8] python3Packages.pytestCheckHook: append disableTests flags after disabledTestPaths flags --- .../interpreters/python/hooks/pytest-check-hook.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh b/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh index ff95f73ba20c8..1696b180b3f65 100644 --- a/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh +++ b/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh @@ -9,10 +9,6 @@ function pytestCheckPhase() { # Compose arguments local -a flagsArray=(-m pytest) - if [ -n "${disabledTests[*]-}" ]; then - disabledTestsString="not $(concatStringsSep " and not " disabledTests)" - flagsArray+=(-k "$disabledTestsString") - fi local -a _pathsArray=() concatTo _pathsArray disabledTestPaths @@ -30,6 +26,11 @@ EOF flagsArray+=("--ignore-glob=$path") done + if [ -n "${disabledTests[*]-}" ]; then + disabledTestsString="not $(concatStringsSep " and not " disabledTests)" + flagsArray+=(-k "$disabledTestsString") + fi + # Compatibility layer to the obsolete pytestFlagsArray eval "flagsArray+=(${pytestFlagsArray[*]-})" From 16f8b5074246aa58305b219247650f29210d579d Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Sun, 2 Mar 2025 07:55:46 +0800 Subject: [PATCH 2/8] python3Packages.pytestCheckHook: support deselecting tests via disabledTestPaths Pass disabledTestPaths elements containing double colons (::) to --deselect instead of --ignore-glob. --- pkgs/development/interpreters/python/hooks/default.nix | 6 ++++++ .../interpreters/python/hooks/pytest-check-hook.sh | 10 +++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/pkgs/development/interpreters/python/hooks/default.nix b/pkgs/development/interpreters/python/hooks/default.nix index d8b8f661ee2ac..8d6c26e7b7f7e 100644 --- a/pkgs/development/interpreters/python/hooks/default.nix +++ b/pkgs/development/interpreters/python/hooks/default.nix @@ -184,6 +184,12 @@ in ] ++ previousPythonAttrs.disabledTestPaths or [ ]; }) ); + disabledTestPaths-item = objprint.overridePythonAttrs (previousPythonAttrs: { + pname = "test-pytestCheckHook-disabledTestPaths-item-${previousPythonAttrs.pname}"; + disabledTestPaths = [ + "tests/test_basic.py::TestBasic" + ] ++ previousPythonAttrs.disabledTestPaths or [ ]; + }); disabledTestPaths-glob = objprint.overridePythonAttrs (previousPythonAttrs: { pname = "test-pytestCheckHook-disabledTestPaths-glob-${previousPythonAttrs.pname}"; disabledTestPaths = [ diff --git a/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh b/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh index 1696b180b3f65..50757ec5b7aec 100644 --- a/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh +++ b/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh @@ -13,8 +13,11 @@ function pytestCheckPhase() { local -a _pathsArray=() concatTo _pathsArray disabledTestPaths for path in "${_pathsArray[@]}"; do - # Check if every path glob matches at least one path - @pythonCheckInterpreter@ - "$path" < Date: Sun, 2 Mar 2025 18:19:18 +0800 Subject: [PATCH 3/8] python3Packages.pytestCheckHook: support enabledTestPaths --- .../interpreters/python/hooks/default.nix | 49 +++++++++++++++++++ .../python/hooks/pytest-check-hook.sh | 28 ++++++++++- .../python/mk-python-derivation.nix | 14 ++++++ 3 files changed, 90 insertions(+), 1 deletion(-) diff --git a/pkgs/development/interpreters/python/hooks/default.nix b/pkgs/development/interpreters/python/hooks/default.nix index 8d6c26e7b7f7e..f0207b570b943 100644 --- a/pkgs/development/interpreters/python/hooks/default.nix +++ b/pkgs/development/interpreters/python/hooks/default.nix @@ -204,6 +204,55 @@ in ] ++ previousPythonAttrs.disabledTestPaths or [ ]; }) ); + enabledTestPaths = objprint.overridePythonAttrs (previousPythonAttrs: { + pname = "test-pytestCheckHook-enabledTestPaths-${previousPythonAttrs.pname}"; + enabledTestPaths = [ + "tests/test_basic.py" + ] ++ previousPythonAttrs.enabledTestPaths or [ ]; + }); + enabledTestPaths-nonexistent = testers.testBuildFailure ( + objprint.overridePythonAttrs (previousPythonAttrs: { + pname = "test-pytestCheckHook-enabledTestPaths-nonexistent-${previousPythonAttrs.pname}"; + enabledTestPaths = [ + "tests/test_foo.py" + ] ++ previousPythonAttrs.enabledTestPaths or [ ]; + }) + ); + enabledTestPaths-dir = objprint.overridePythonAttrs (previousPythonAttrs: { + pname = "test-pytestCheckHook-enabledTestPaths-dir-${previousPythonAttrs.pname}"; + enabledTestPaths = [ + "tests" + ] ++ previousPythonAttrs.enabledTestPaths or [ ]; + }); + enabledTestPaths-dir-disabledTestPaths = objprint.overridePythonAttrs (previousPythonAttrs: { + pname = "test-pytestCheckHook-enabledTestPaths-dir-disabledTestPaths-${previousPythonAttrs.pname}"; + enabledTestPaths = [ + "tests" + ] ++ previousPythonAttrs.enabledTestPaths or [ ]; + disabledTestPaths = [ + "tests/test_basic.py" + ] ++ previousPythonAttrs.disabledTestPaths or [ ]; + }); + enabledTestPaths-glob = objprint.overridePythonAttrs (previousPythonAttrs: { + pname = "test-pytestCheckHook-enabledTestPaths-glob-${previousPythonAttrs.pname}"; + enabledTestPaths = [ + "tests/test_obj*.py" + ] ++ previousPythonAttrs.enabledTestPaths or [ ]; + }); + enabledTestPaths-glob-nonexistent = testers.testBuildFailure ( + objprint.overridePythonAttrs (previousPythonAttrs: { + pname = "test-pytestCheckHook-enabledTestPaths-glob-nonexistent-${previousPythonAttrs.pname}"; + enabledTestPaths = [ + "tests/test_foo*.py" + ] ++ previousPythonAttrs.enabledTestPaths or [ ]; + }) + ); + enabledTestPaths-item = objprint.overridePythonAttrs (previousPythonAttrs: { + pname = "test-pytestCheckHook-enabledTestPaths-item-${previousPythonAttrs.pname}"; + enabledTestPaths = [ + "tests/test_basic.py::TestBasic" + ] ++ previousPythonAttrs.enabledTestPaths or [ ]; + }); }; }; } ./pytest-check-hook.sh diff --git a/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh b/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh index 50757ec5b7aec..afe2c06567982 100644 --- a/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh +++ b/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh @@ -10,7 +10,33 @@ function pytestCheckPhase() { # Compose arguments local -a flagsArray=(-m pytest) - local -a _pathsArray=() + local -a _pathsArray + local path + + _pathsArray=() + concatTo _pathsArray enabledTestPaths + for path in "${_pathsArray[@]}"; do + if [[ "$path" =~ "::" ]]; then + flagsArray+=("$path") + else + # The `|| kill "$$"` trick propagates the errors from the process substitutiton subshell, + # which is suggested by a StackOverflow answer: https://unix.stackexchange.com/a/217643 + readarray -t -O"${#flagsArray[@]}" flagsArray < <(@pythonCheckInterpreter@ - "$path" < Date: Sun, 9 Mar 2025 22:38:50 +0800 Subject: [PATCH 4/8] python3Packages.pytestCheckHook: Make disabledTests support sub-expressions Wrap each elements of disabledTests with parenthesis so that when __structuredAttrs = true, people could use sub-expressions an element. E.g. ```nix { disabledTests = [ "ClassFoo and test_foo" "test_bar" ]; } --- pkgs/development/interpreters/python/hooks/default.nix | 8 ++++++++ .../interpreters/python/hooks/pytest-check-hook.sh | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pkgs/development/interpreters/python/hooks/default.nix b/pkgs/development/interpreters/python/hooks/default.nix index f0207b570b943..1d5eb5c2c720d 100644 --- a/pkgs/development/interpreters/python/hooks/default.nix +++ b/pkgs/development/interpreters/python/hooks/default.nix @@ -170,6 +170,14 @@ in "test_print" ] ++ previousPythonAttrs.disabledTests or [ ]; }); + disabledTests-expression = objprint.overridePythonAttrs (previousPythonAttrs: { + __structuredAttrs = true; + pname = "test-pytestCheckHook-disabledTests-expression-${previousPythonAttrs.pname}"; + disabledTests = [ + "TestBasic and test_print" + "test_str" + ] ++ previousPythonAttrs.disabledTests or [ ]; + }); disabledTestPaths = objprint.overridePythonAttrs (previousPythonAttrs: { pname = "test-pytestCheckHook-disabledTestPaths-${previousPythonAttrs.pname}"; disabledTestPaths = [ diff --git a/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh b/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh index afe2c06567982..61082a771c0f7 100644 --- a/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh +++ b/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh @@ -57,7 +57,8 @@ EOF done if [ -n "${disabledTests[*]-}" ]; then - disabledTestsString="not $(concatStringsSep " and not " disabledTests)" + # not (keyword1) and not (keyword2) + disabledTestsString="not ($(concatStringsSep ") and not (" disabledTests))" flagsArray+=(-k "$disabledTestsString") fi From f7aeb5a9b72806b9b765490990375de5dc20d8f4 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Sun, 2 Mar 2025 21:31:34 +0800 Subject: [PATCH 5/8] python3Packages.pytestCheckHook: support enabledTests --- .../interpreters/python/hooks/default.nix | 23 +++++++++++++++++++ .../python/hooks/pytest-check-hook.sh | 23 +++++++++++++++---- .../python/mk-python-derivation.nix | 8 ++++--- 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/pkgs/development/interpreters/python/hooks/default.nix b/pkgs/development/interpreters/python/hooks/default.nix index 1d5eb5c2c720d..d80013fa212da 100644 --- a/pkgs/development/interpreters/python/hooks/default.nix +++ b/pkgs/development/interpreters/python/hooks/default.nix @@ -212,6 +212,29 @@ in ] ++ previousPythonAttrs.disabledTestPaths or [ ]; }) ); + enabledTests = objprint.overridePythonAttrs (previousPythonAttrs: { + pname = "test-pytestCheckHook-enabledTests-${previousPythonAttrs.pname}"; + enabledTests = [ + "TestBasic" + ] ++ previousPythonAttrs.disabledTests or [ ]; + }); + enabledTests-expression = objprint.overridePythonAttrs (previousPythonAttrs: { + __structuredAttrs = true; + pname = "test-pytestCheckHook-enabledTests-expression-${previousPythonAttrs.pname}"; + enabledTests = [ + "TestBasic and test_print" + "test_str" + ] ++ previousPythonAttrs.disabledTests or [ ]; + }); + enabledTests-disabledTests = objprint.overridePythonAttrs (previousPythonAttrs: { + pname = "test-pytestCheckHook-enabledTests-disabledTests-${previousPythonAttrs.pname}"; + enabledTests = [ + "TestBasic" + ] ++ previousPythonAttrs.disabledTests or [ ]; + disabledTests = [ + "test_print" + ] ++ previousPythonAttrs.disabledTests or [ ]; + }); enabledTestPaths = objprint.overridePythonAttrs (previousPythonAttrs: { pname = "test-pytestCheckHook-enabledTestPaths-${previousPythonAttrs.pname}"; enabledTestPaths = [ diff --git a/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh b/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh index 61082a771c0f7..f728e9e9b26ba 100644 --- a/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh +++ b/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh @@ -3,6 +3,23 @@ echo "Sourcing pytest-check-hook" +function _pytestIncludeExcludeExpr() { + local includeListName="$1" + local -n includeListRef="$includeListName" + local excludeListName="$2" + local -n excludeListRef="$excludeListName" + local includeString excludeString + if [[ -n "${includeListRef[*]-}" ]]; then + # ((element1) or (element2)) + includeString="(($(concatStringsSep ") or (" "$includeListName")))" + fi + if [[ -n "${excludeListRef[*]-}" ]]; then + # and not (element1) and not (element2) + excludeString="${includeString:+ and }not ($(concatStringsSep ") and not (" "$excludeListName"))" + fi + echo "$includeString$excludeString" +} + function pytestCheckPhase() { echo "Executing pytestCheckPhase" runHook preCheck @@ -56,10 +73,8 @@ EOF fi done - if [ -n "${disabledTests[*]-}" ]; then - # not (keyword1) and not (keyword2) - disabledTestsString="not ($(concatStringsSep ") and not (" disabledTests))" - flagsArray+=(-k "$disabledTestsString") + if [[ -n "${enabledTests[*]-}" ]] || [[ -n "${disabledTests[*]-}" ]]; then + flagsArray+=(-k "$(_pytestIncludeExcludeExpr enabledTests disabledTests)") fi # Compatibility layer to the obsolete pytestFlagsArray diff --git a/pkgs/development/interpreters/python/mk-python-derivation.nix b/pkgs/development/interpreters/python/mk-python-derivation.nix index b2f47b75e0272..43bcaa59fc379 100644 --- a/pkgs/development/interpreters/python/mk-python-derivation.nix +++ b/pkgs/development/interpreters/python/mk-python-derivation.nix @@ -109,6 +109,7 @@ let "disabledTestPaths" "disabledTests" "enabledTestPaths" + "enabledTests" "pytestFlags" "pytestFlagsArray" "unittestFlags" @@ -451,13 +452,14 @@ let lib.mapAttrs ( name: value: - lib.throwIf (attrs.${name} == [ ]) - "${lib.getName finalAttrs}: ${name} must be unspecified, null or a non-empty list." - attrs.${name} + lib.throwIf ( + attrs.${name} == [ ] + ) "${lib.getName finalAttrs}: ${name} must be unspecified, null or a non-empty list." attrs.${name} ) ( getOptionalAttrs [ "enabledTestPaths" + "enabledTests" ] attrs ) ) From e1e072bf0395c722735975293863df8601cebbef Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Sun, 2 Mar 2025 21:36:07 +0800 Subject: [PATCH 6/8] python3Packages.pytestCheckHook: support enabledTestMarks and disabledTestMarks --- .../interpreters/python/hooks/pytest-check-hook.sh | 4 ++++ pkgs/development/interpreters/python/mk-python-derivation.nix | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh b/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh index f728e9e9b26ba..bbdc066d48b24 100644 --- a/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh +++ b/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh @@ -77,6 +77,10 @@ EOF flagsArray+=(-k "$(_pytestIncludeExcludeExpr enabledTests disabledTests)") fi + if [[ -n "${enabledTestMarks[*]-}" ]] || [[ -n "${disabledTestMarks[*]-}" ]]; then + flagsArray+=(-m "$(_pytestIncludeExcludeExpr enabledTestMarks disabledTestMarks)") + fi + # Compatibility layer to the obsolete pytestFlagsArray eval "flagsArray+=(${pytestFlagsArray[*]-})" diff --git a/pkgs/development/interpreters/python/mk-python-derivation.nix b/pkgs/development/interpreters/python/mk-python-derivation.nix index 43bcaa59fc379..b5aa006ba2124 100644 --- a/pkgs/development/interpreters/python/mk-python-derivation.nix +++ b/pkgs/development/interpreters/python/mk-python-derivation.nix @@ -106,8 +106,10 @@ let "doInstallCheck" "pyproject" "format" + "disabledTestMarks" "disabledTestPaths" "disabledTests" + "enabledTestMarks" "enabledTestPaths" "enabledTests" "pytestFlags" @@ -441,6 +443,7 @@ let } // optionalAttrs (attrs.doCheck or true) ( getOptionalAttrs [ + "disabledTestMarks" "disabledTestPaths" "disabledTests" "pytestFlags" @@ -458,6 +461,7 @@ let ) ( getOptionalAttrs [ + "enabledTestMarks" "enabledTestPaths" "enabledTests" ] attrs From 8ebcc32ce1e48f76103c082e035a78a546f073ab Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Fri, 7 Mar 2025 18:42:32 +0800 Subject: [PATCH 7/8] python3Packages.xyzservices: use disabledTestMarks --- pkgs/development/python-modules/xyzservices/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/xyzservices/default.nix b/pkgs/development/python-modules/xyzservices/default.nix index 61df3a50d9eff..944997ce28156 100644 --- a/pkgs/development/python-modules/xyzservices/default.nix +++ b/pkgs/development/python-modules/xyzservices/default.nix @@ -24,9 +24,9 @@ buildPythonPackage rec { setuptools-scm ]; - pytestFlagsArray = [ + disabledTestMarks = [ # requires network connections - "-m 'not request'" + "request" ]; pythonImportsCheck = [ "xyzservices.providers" ]; From bb76d746322a52c41feeeef13a7cf577537f8594 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 10 Mar 2025 00:06:34 +0800 Subject: [PATCH 8/8] doc: python.section.md: partial rewrites for pytest and pytestCheckHook Co-authored-by: Sandro Co-authored-by: Matt Sturgeon Co-authored-by: Wolfgang Walther --- doc/languages-frameworks/python.section.md | 168 ++++++++++++--------- 1 file changed, 96 insertions(+), 72 deletions(-) diff --git a/doc/languages-frameworks/python.section.md b/doc/languages-frameworks/python.section.md index bbc5da116a6a8..96dec589dadc9 100644 --- a/doc/languages-frameworks/python.section.md +++ b/doc/languages-frameworks/python.section.md @@ -1233,46 +1233,57 @@ test run would be: However, many repositories' test suites do not translate well to nix's build sandbox, and will generally need many tests to be disabled. -To filter tests using pytest, one can do the following: +This is achievable by +- Including paths or test items (`path/to/file.py::MyClass` or `path/to/file.py::MyClass::test_method`) with positional arguments. +- Excluding paths with `--ignore` or globbed paths with `--ignore-glob`. +- Excluding test items using the `--deselect` flag. +- Including or excluding classes or test methods by their name using the `-k` flag. +- Including or excluding test by their marks using the `-m` flag. -```nix -{ - nativeCheckInputs = [ pytest ]; - # avoid tests which need additional data or touch network - checkPhase = '' - runHook preCheck +We highly recommend `pytestCheckHook` for an easier and more structural setup. - pytest tests/ --ignore=tests/integration -k 'not download and not update' --ignore=tests/test_failing.py +#### Using pytestCheckHook {#using-pytestcheckhook} - runHook postCheck - ''; +`pytestCheckHook` is a convenient hook which will set up (or configure) +a [`checkPhase`](#ssec-check-phase) to run `pytest`. This is also beneficial +when a package may need many items disabled to run the test suite. +Most packages use `pytest` or `unittest`, which is compatible with `pytest`, +so you will most likely use `pytestCheckHook`. + +To use `pytestCheckHook`, add it to `nativeCheckInputs`. +Adding `pytest` is not required, since it is included with `pytestCheckHook`. + +```nix +{ + nativeCheckInputs = [ + pytestCheckHook + ]; } ``` -`--ignore` will tell pytest to ignore that file or directory from being -collected as part of a test run. This is useful is a file uses a package -which is not available in nixpkgs, thus skipping that test file is much -easier than having to create a new package. +`pytestCheckHook` recognizes the following attributes: -`-k` is used to define a predicate for test names. In this example, we are -filtering out tests which contain `download` or `update` in their test case name. -Only one `-k` argument is allowed, and thus a long predicate should be concatenated -with “\\” and wrapped to the next line. +`enabledTestPaths` and `disabledTestPaths` -::: {.note} -In pytest==6.0.1, the use of “\\” to continue a line (e.g. `-k 'not download \'`) has -been removed, in this case, it's recommended to use `pytestCheckHook`. -::: +: To specify path globs (files or directories) or test items. -#### Using pytestCheckHook {#using-pytestcheckhook} +`enabledTests` and `disabledTests` -`pytestCheckHook` is a convenient hook which will set up (or configure) -a [`checkPhase`](#ssec-check-phase) to run `pytest`. This is also beneficial -when a package may need many items disabled to run the test suite. -Most packages use `pytest` or `unittest`, which is compatible with `pytest`, -so you will most likely use `pytestCheckHook`. +: To specify keywords for class names or test method names. + +`enabledTestMarks` and `disabledTestMarks` + +: To specify test marks. -Using the example above, the analogous `pytestCheckHook` usage would be: +`pytestFlags` + +: To append additional command-line arguments to `pytest`. + +By default, `pytest` automatically discovers which tests to run. +If tests are explicitly enabled, only those tests will run. +A test, that is both enabled and disabled, will not run. + +The following example demonstrates usage of various `pytestCheckHook` attributes: ```nix { @@ -1280,26 +1291,73 @@ Using the example above, the analogous `pytestCheckHook` usage would be: pytestCheckHook ]; - # requires additional data - pytestFlags = [ + # Allow running the following test paths and test objects. + enabledTestPaths = [ + # Find tests under the tests directory. + # The trailing slash is not necessary. "tests/" - "--ignore=tests/integration" + + # Additionally run test_foo + "other-tests/test_foo.py::Foo::test_foo" ]; + # Override the above-enabled test paths and test objects. + disabledTestPaths = [ + # Tests under tests/integration requires additional data. + "tests/integration" + ]; + + # Allow tests by keywords matching their class names or method names. + enabledTests = [ + # pytest by default only runs test methods begin with "test_" or end with "_test". + # This includes all functions whose name contains "test". + "test" + ]; + + # Override the above-enabled tests by keywords matching their class names or method names. disabledTests = [ - # touches network + # Tests touching networks. + "upload" "download" - "update" ]; - disabledTestPaths = [ - "tests/test_failing.py" + # Additional pytest flags + pytestFlags = [ + # Disable benchmarks and run benchmarking tests only once. + "--benchmark-disable" + ]; +} +``` + +These attributes are all passed into the derivation directly +and added to the `pytest` command without additional Bash expansion. +It requires `__structuredAttrs = true` to pass list elements containing spaces. + +The `TestsPaths` attributes expand Unix-style globs. +If a test path contains characters like `*`, `?`, `[`, or `]`, you can +quote them with square brackets (`[*]`, `[?]`, `[[]`, and `[]]`) to match literally. + +The `Tests` and `TestMarks` attribute pairs +form a logical expression `((included_element1) or (included_element2)) and not (excluded_element1) and not (excluded_element2)` +which will be passed to pytest's `-k` and `-m` flags respectively. +With `__structuredAttrs = true` enabled, they additionally support sub-expressions. + +For example, you could disable test items like `TestFoo::test_bar_functionality` +by disabling tests that match both `"Foo"` **and** `"bar"`: + +```nix +{ + __structuredAttrs = true; + + disabledTests = [ + "Foo and bar" ]; } ``` -This is especially useful when tests need to be conditionally disabled, -for example: +The main benefits of using `pytestCheckHook` to construct `pytest` commands +is structuralization and eval-time accessibility. +This is especially helpful to select tests or specify flags conditionally: ```nix { @@ -1317,10 +1375,6 @@ for example: } ``` -Trying to concatenate the related strings to disable tests in a regular -[`checkPhase`](#ssec-check-phase) would be much harder to read. This also enables us to comment on -why specific tests are disabled. - #### Using pythonImportsCheck {#using-pythonimportscheck} Although unit tests are highly preferred to validate correctness of a package, not @@ -2008,36 +2062,6 @@ Occasionally packages don't make use of a common test framework, which may then #### Common issues {#common-issues} -* Non-working tests can often be deselected. Most Python modules - do follow the standard test protocol where the pytest runner can be used. - `pytest` supports the `-k` and `--ignore-glob` parameters to ignore test - methods or classes as well as whole files. For `pytestCheckHook` these are - conveniently exposed as `disabledTests` and `disabledTestPaths` respectively. - - ```nix - buildPythonPackage { - # ... - nativeCheckInputs = [ - pytestCheckHook - ]; - - disabledTests = [ - "function_name" - "other_function" - ]; - - disabledTestPaths = [ - "path/to/performance.py" - "path/to/connect-*.py" - ]; - } - ``` - - ::: {.note} - If the test path to disable contains characters like `*`, `?`, `[`, and `]`, - quote them with square brackets (`[*]`, `[?]`, `[[]`, and `[]]`) to match literally. - ::: - * Tests that attempt to access `$HOME` can be fixed by using the following work-around before running tests (e.g. `preCheck`): `export HOME=$(mktemp -d)` * Compiling with Cython causes tests to fail with a `ModuleNotLoadedError`.