re-enable pythran on windows; test skip hyghiene

recipe/bld.bat

@@ -12,8 +12,6 @@ echo %LIBRARY_LIB%\lapack.lib > %LIBRARY_LIB%\lapack.cobjects
 REM Use the G77 ABI wrapper everywhere so that the underlying blas implementation
 REM can have a G77 ABI (currently only MKL)
 set SCIPY_USE_G77_ABI_WRAPPER=1
-REM TODO: remove this again ASAP
-set SCIPY_USE_PYTHRAN=0
 
 REM This builds a Fortran file which calls a C function, but numpy.distutils
 REM creates an isolated DLL for these fortran functions and therefore it doesn't

recipe/meta.yaml

@@ -14,6 +14,11 @@ source:
   # the submodules (not in tarball due to dear-github/dear-github#214)
   - url: https://github.com/scipy/scipy/archive/refs/tags/v{{ version }}.tar.gz
     sha256: 0c5a87bfa416d38f0ab33c8604703db50b74034d0631cae5777dbda4ce365c86
+    patches:
+      # backport scipy/scipy#15531 to fix pythran on windows
+      - patches/0001-BUG-Fix-importing-scipy.lib._pep440.patch
+      # backport scipy/scipy#15532 to fix timeout of a long-running test
+      - patches/0002-parametrize-test_ldl_type_size_combinations.patch
   # https://github.com/scipy/scipy/tree/v{{ version }}/scipy/_lib
   - git_url: https://github.com/scipy/boost-headers-only.git
     git_rev: 2110ce20e59917b85b12059b3ddd8b133549f662
@@ -27,7 +32,7 @@ source:
     folder: scipy/sparse/linalg/_propack/PROPACK
 
 build:
-  number: 0
+  number: 1
   skip: true  # [py<=37]
 
 requirements:
@@ -65,14 +70,17 @@ requirements:
 {% set tests_to_skip = "_not_a_real_test" %}
 # skip a test that fails with MKL + AVX512 (non-AVX512 passes)
 {% set tests_to_skip = tests_to_skip + " or test_x0_equals_Mb[bicgstab]" %}
-# skip some tests that hang/fail on pypy
-{% if python_impl == "pypy" %}
-  # hangs
-  {% set tests_to_skip = tests_to_skip + " or test_mixed_threads_processes" %}  # [aarch64 or ppc64le]
-  # failures
-  {% set tests_to_skip = tests_to_skip + " or test_cdist_refcount" %}           # [unix]
-  {% set tests_to_skip = tests_to_skip + " or test_boost_eval_issue_14606" %}   # [unix]
-{% endif %}
+# these multithreading tests occasionally hang in emulation
+{% set tests_to_skip = tests_to_skip + " or test_immediate_updating" %}         # [aarch64 or ppc64le]
+{% set tests_to_skip = tests_to_skip + " or test_mixed_threads_processes" %}    # [aarch64 or ppc64le]
+{% set tests_to_skip = tests_to_skip + " or test_multiprocess[fft]" %}          # [aarch64 or ppc64le]
+{% set tests_to_skip = tests_to_skip + " or (test__differential_evolution and test_parallel)" %}  # [aarch64 or ppc64le]
+# on the slowest agents, these tests take more than 20min in emulation
+{% set tests_to_skip = tests_to_skip + " or (test_lsq_linear and test_large_rank_deficient)" %}   # [aarch64 or ppc64le]
+# needs one or two gc.collect()
+{% set tests_to_skip = tests_to_skip + " or test_cdist_refcount" %}             # [python_impl == "pypy"]
+# unclear why a warning isn't raised. Might be related to scipy/scipy#15121
+{% set tests_to_skip = tests_to_skip + " or test_boost_eval_issue_14606" %}     # [python_impl == "pypy"]
 
 test:
   requires:
@@ -87,13 +95,10 @@ test:
     # since this takes much longer, do not run the most heavy-weight tests, i.e. use
     # label='fast'; same for PyPy due to huge runtime; otherwise set label='full'
     {% set label = "'fast'" if (aarch64 or ppc64le or python_impl == "pypy") else "'full'" %}
-    # aarch tests ran in reduced form on drone due to timing constraints;
-    # going back to the full test suite will need some debugging of hangs
-    {% set tests = "['scipy.linalg', 'scipy.fft']" if aarch64 else "None" %}
     # for signature of scipy.test see below (note default: label='fast'):
     # https://github.com/scipy/scipy/blob/v1.7.0/scipy/_lib/_testutils.py#L27
-    {% set param = "verbose=2, label=" + label + ", tests=" + tests %}
-    {% set extra = "extra_argv=['-k', 'not (" + tests_to_skip + ")', '--timeout=2400', '--durations=50']" %}
+    {% set param = "verbose=1, label=" + label + ", tests=None" %}
+    {% set extra = "extra_argv=['-k', 'not (" + tests_to_skip + ")', '-nauto', '--timeout=1200', '--durations=50']" %}
     - python -c "import scipy, sys; sys.exit(not scipy.test({{ param }}, {{ extra }}))"  # [not ppc64le]
     # NOTE: test suite is skipped on ppc due to bugs in QEMU code that cause
     # CI to fail, even though the tests would run through on native hardware

recipe/patches/0001-BUG-Fix-importing-scipy.lib._pep440.patch

@@ -0,0 +1,62 @@
+From f324c122629ee07c7fef56496b8efb331fdaf0c8 Mon Sep 17 00:00:00 2001
+From: Isuru Fernando <isuruf@gmail.com>
+Date: Sun, 6 Feb 2022 01:29:31 -0600
+Subject: [PATCH 1/2] [BUG] Fix importing scipy.lib._pep440
+
+- Change scipy/_lib/_pep440 to scipy._lib._pep440
+- Do the second import in else clause to avoid errors like this
+  in the future
+- Set the imported module to a variable
+---
+ setup.py | 14 ++++++++------
+ 1 file changed, 8 insertions(+), 6 deletions(-)
+
+diff --git a/setup.py b/setup.py
+index a5ccaa159..e7ba1c45a 100755
+--- a/setup.py
++++ b/setup.py
+@@ -232,16 +232,17 @@ def get_build_ext_override():
+         try:
+             import pythran
+             from pythran.dist import PythranBuildExt
++        except ImportError:
++            BaseBuildExt = npy_build_ext
++        else:
+             BaseBuildExt = PythranBuildExt[npy_build_ext]
+-            importlib.import_module('scipy/_lib/_pep440')
++            _pep440 = importlib.import_module('scipy._lib._pep440')
+             if _pep440.parse(pythran.__version__) < _pep440.Version('0.10.0'):
+                 raise RuntimeError("The installed `pythran` is too old, >= "
+                                    "0.10.0 is needed, {} detected. Please "
+                                    "upgrade Pythran, or use `export "
+                                    "SCIPY_USE_PYTHRAN=0`.".format(
+                                    pythran.__version__))
+-        except ImportError:
+-            BaseBuildExt = npy_build_ext
+     else:
+         BaseBuildExt = npy_build_ext
+
+@@ -345,7 +346,10 @@ def generate_cython():
+         try:
+             # Note, pip may not be installed or not have been used
+             import pip
+-            importlib.import_module('scipy/_lib/_pep440')
++        except (ImportError, ModuleNotFoundError):
++            raise RuntimeError("Running cythonize failed!")
++        else:
++            _pep440 = importlib.import_module('scipy._lib._pep440')
+             if _pep440.parse(pip.__version__) < _pep440.Version('18.0.0'):
+                 raise RuntimeError("Cython not found or too old. Possibly due "
+                                    "to `pip` being too old, found version {}, "
+@@ -353,8 +357,6 @@ def generate_cython():
+                                    pip.__version__))
+             else:
+                 raise RuntimeError("Running cythonize failed!")
+-        except (ImportError, ModuleNotFoundError):
+-            raise RuntimeError("Running cythonize failed!")
+
+
+ def parse_setuppy_commands():
+-- 
+2.32.0.windows.2
+

recipe/patches/0002-parametrize-test_ldl_type_size_combinations.patch

@@ -0,0 +1,126 @@
+From 2b6c76423e9748369de0d12c6106df594d83fc22 Mon Sep 17 00:00:00 2001
+From: "H. Vetinari" <h.vetinari@gmx.com>
+Date: Sun, 6 Feb 2022 19:09:53 +1100
+Subject: [PATCH 2/2] * parametrize test_ldl_type_size_combinations
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+  best viewed without whitespace changes, e.g.
+  git show -b <hash_of_this_commit> or using ⚙️ in GH
+
+* reduce size in test_ldl_type_size_combinations_*
+
+  Originally, this seemed like the best way to capture failures
+  but these days, it is overkill and can be reduced
+
+  Suggested-By: Ilhan Polat <ilhanpolat@gmail.com>
+---
+ scipy/linalg/tests/test_decomp_ldl.py | 86 ++++++++++++++-------------
+ 1 file changed, 44 insertions(+), 42 deletions(-)
+
+diff --git a/scipy/linalg/tests/test_decomp_ldl.py b/scipy/linalg/tests/test_decomp_ldl.py
+index ce6b7c05f..3f08e529a 100644
+--- a/scipy/linalg/tests/test_decomp_ldl.py
++++ b/scipy/linalg/tests/test_decomp_ldl.py
+@@ -5,6 +5,7 @@ from numpy import (array, eye, zeros, empty_like, empty, tril_indices_from,
+                    complex64, complex128)
+ from numpy.random import rand, randint, seed
+ from scipy.linalg import ldl
++import pytest
+ from pytest import raises as assert_raises, warns
+ from numpy import ComplexWarning
+
+@@ -90,46 +91,47 @@ def test_permutations():
+         assert_(not any(l[p, :][u_ind]), 'Spin {} failed'.format(_))
+
+
+-def test_ldl_type_size_combinations():
++@pytest.mark.parametrize("dtype", [float32, float64])
++@pytest.mark.parametrize("n", [30, 150])
++def test_ldl_type_size_combinations_real(n, dtype):
+     seed(1234)
+-    sizes = [30, 750]
+-    real_dtypes = [float32, float64]
+-    complex_dtypes = [complex64, complex128]
+-
+-    for n, dtype in itertools.product(sizes, real_dtypes):
+-        msg = ("Failed for size: {}, dtype: {}".format(n, dtype))
+-
+-        x = rand(n, n).astype(dtype)
+-        x = x + x.T
+-        x += eye(n, dtype=dtype)*dtype(randint(5, 1e6))
+-
+-        l, d1, p = ldl(x)
+-        u, d2, p = ldl(x, lower=0)
+-        rtol = 1e-4 if dtype is float32 else 1e-10
+-        assert_allclose(l.dot(d1).dot(l.T), x, rtol=rtol, err_msg=msg)
+-        assert_allclose(u.dot(d2).dot(u.T), x, rtol=rtol, err_msg=msg)
+-
+-    for n, dtype in itertools.product(sizes, complex_dtypes):
+-        msg1 = ("Her failed for size: {}, dtype: {}".format(n, dtype))
+-        msg2 = ("Sym failed for size: {}, dtype: {}".format(n, dtype))
+-
+-        # Complex hermitian upper/lower
+-        x = (rand(n, n)+1j*rand(n, n)).astype(dtype)
+-        x = x+x.conj().T
+-        x += eye(n, dtype=dtype)*dtype(randint(5, 1e6))
+-
+-        l, d1, p = ldl(x)
+-        u, d2, p = ldl(x, lower=0)
+-        rtol = 1e-4 if dtype is complex64 else 1e-10
+-        assert_allclose(l.dot(d1).dot(l.conj().T), x, rtol=rtol, err_msg=msg1)
+-        assert_allclose(u.dot(d2).dot(u.conj().T), x, rtol=rtol, err_msg=msg1)
+-
+-        # Complex symmetric upper/lower
+-        x = (rand(n, n)+1j*rand(n, n)).astype(dtype)
+-        x = x+x.T
+-        x += eye(n, dtype=dtype)*dtype(randint(5, 1e6))
+-
+-        l, d1, p = ldl(x, hermitian=0)
+-        u, d2, p = ldl(x, lower=0, hermitian=0)
+-        assert_allclose(l.dot(d1).dot(l.T), x, rtol=rtol, err_msg=msg2)
+-        assert_allclose(u.dot(d2).dot(u.T), x, rtol=rtol, err_msg=msg2)
++    msg = ("Failed for size: {}, dtype: {}".format(n, dtype))
++
++    x = rand(n, n).astype(dtype)
++    x = x + x.T
++    x += eye(n, dtype=dtype)*dtype(randint(5, 1e6))
++
++    l, d1, p = ldl(x)
++    u, d2, p = ldl(x, lower=0)
++    rtol = 1e-4 if dtype is float32 else 1e-10
++    assert_allclose(l.dot(d1).dot(l.T), x, rtol=rtol, err_msg=msg)
++    assert_allclose(u.dot(d2).dot(u.T), x, rtol=rtol, err_msg=msg)
++
++
++@pytest.mark.parametrize("dtype", [complex64, complex128])
++@pytest.mark.parametrize("n", [30, 150])
++def test_ldl_type_size_combinations_complex(n, dtype):
++    seed(1234)
++    msg1 = ("Her failed for size: {}, dtype: {}".format(n, dtype))
++    msg2 = ("Sym failed for size: {}, dtype: {}".format(n, dtype))
++
++    # Complex hermitian upper/lower
++    x = (rand(n, n)+1j*rand(n, n)).astype(dtype)
++    x = x+x.conj().T
++    x += eye(n, dtype=dtype)*dtype(randint(5, 1e6))
++
++    l, d1, p = ldl(x)
++    u, d2, p = ldl(x, lower=0)
++    rtol = 1e-4 if dtype is complex64 else 1e-10
++    assert_allclose(l.dot(d1).dot(l.conj().T), x, rtol=rtol, err_msg=msg1)
++    assert_allclose(u.dot(d2).dot(u.conj().T), x, rtol=rtol, err_msg=msg1)
++
++    # Complex symmetric upper/lower
++    x = (rand(n, n)+1j*rand(n, n)).astype(dtype)
++    x = x+x.T
++    x += eye(n, dtype=dtype)*dtype(randint(5, 1e6))
++
++    l, d1, p = ldl(x, hermitian=0)
++    u, d2, p = ldl(x, lower=0, hermitian=0)
++    assert_allclose(l.dot(d1).dot(l.T), x, rtol=rtol, err_msg=msg2)
++    assert_allclose(u.dot(d2).dot(u.T), x, rtol=rtol, err_msg=msg2)
+-- 
+2.32.0.windows.2
+