ENH: stats: add random variable infrastructure

.circleci/config.yml

@@ -42,12 +42,6 @@ commands:
           command: |
             echo $(git log -1 --pretty=%B) | tee gitlog.txt
             echo ${CI_PULL_REQUEST//*pull\//} | tee merge.txt
-            if [[ $(cat merge.txt) != "" ]]; then
-              echo "Merging $(cat merge.txt)";
-              git remote add upstream https://github.com/scipy/scipy.git;
-              git pull --ff-only upstream "refs/pull/$(cat merge.txt)/merge";
-              git fetch upstream main;
-            fi
 
 jobs:
 # Build SciPy from source

.github/ISSUE_TEMPLATE/config.yml

@@ -3,9 +3,9 @@ contact_links:
   - name: Stack Overflow
     url: https://stackoverflow.com/questions/tagged/scipy
     about: Please ask and answer usage questions on Stack Overflow
-  - name: Developer Mailing list
-    url: https://mail.python.org/mailman3/lists/scipy-dev.python.org/
-    about: Development discussions and announcements on the mailing list
+  - name: Developer Forum
+    url: https://discuss.scientific-python.org/c/contributor/scipy
+    about: Development discussions and announcements on the forum
   - name: Blank issue
     url: https://github.com/scipy/scipy/issues/new
     about: Please note that other templates should be used in most cases

.github/workflows/array_api.yml

@@ -78,7 +78,7 @@ jobs:
         echo "timestamp=${NOW}" >> $GITHUB_OUTPUT
 
     - name: Setup compiler cache
-      uses:  actions/cache@v3
+      uses:  actions/cache@v4
       id:    cache-ccache
       with:
         path: ${{ steps.prep-ccache.outputs.dir }}
@@ -100,8 +100,4 @@ jobs:
         python dev.py --no-build test -b all -t scipy.special.tests.test_support_alternative_backends -- --durations 3 --timeout=60
         python dev.py --no-build test -b all -t scipy._lib.tests.test_array_api -- --durations 3 --timeout=60
         python dev.py --no-build test -b all -t scipy._lib.tests.test__util -- --durations 3 --timeout=60
-        python dev.py --no-build test -b all -t scipy.stats.tests.test_stats -- --durations 3 --timeout=60
-        python dev.py --no-build test -b all -t scipy.stats.tests.test_morestats -- --durations 3 --timeout=60
-        python dev.py --no-build test -b all -t scipy.stats.tests.test_variation -- --durations 3 --timeout=60
-        python dev.py --no-build test -b all -t scipy.stats.tests.test_resampling -- --durations 3 --timeout=60
-        python dev.py --no-build test -b all -t scipy.optimize.tests.test_chandrupatla -- --durations 3 --timeout=60
+        python dev.py --no-build test -b all -s stats -- --durations 3 --timeout=60

.github/workflows/free_threaded_wheels.yml

@@ -0,0 +1,160 @@
+# Workflow to build and test wheels for the free-threaded Python build.
+#
+# This should be merged back into wheels.yml when free-threaded wheel
+# builds can be uploaded to pypi along with the rest of scipy's release
+# artifacts.
+#
+# To work on the wheel building infrastructure on a fork, comment out:
+#
+# if: github.repository == 'scipy/scipy'
+#
+# in the get_commit_message job. Be sure to include [wheel build] in your commit
+# message to trigger the build. All files related to wheel building are located
+# at tools/wheels/
+name: Free-Threaded Wheel Builder
+
+on:
+    schedule:
+    #        ┌───────────── minute (0 - 59)
+    #        │  ┌───────────── hour (0 - 23)
+    #        │  │ ┌───────────── day of the month (1 - 31)
+    #        │  │ │ ┌───────────── month (1 - 12 or JAN-DEC)
+    #        │  │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
+    #        │  │ │ │ │
+    - cron: "9  9 * * *"
+    push:
+      branches:
+        - maintenance/**
+    pull_request:
+      branches:
+        - main
+        - maintenance/**
+    workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read # to fetch code (actions/checkout)
+
+jobs:
+  get_commit_message:
+    name: Get commit message
+    runs-on: ubuntu-latest
+    if: github.repository == 'scipy/scipy'
+    outputs:
+      message: ${{ steps.commit_message.outputs.message }}
+    steps:
+      - name: Checkout scipy
+        uses: actions/[email protected]
+        # Gets the correct commit message for pull request
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: Get commit message
+        id: commit_message
+        run: |
+          set -xe
+          COMMIT_MSG=$(git log --no-merges -1)
+          RUN="0"
+          if [[ "$COMMIT_MSG" == *"[wheel build]"* ]]; then
+              RUN="1"
+          fi
+          echo "message=$RUN" >> $GITHUB_OUTPUT
+          echo github.ref ${{ github.ref }}
+
+  build_wheels:
+    name: Wheel, ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
+      ${{ matrix.buildplat[2] }} ${{ matrix.buildplat[3] }}
+      ${{ matrix.buildplat[4] }}
+    needs: get_commit_message
+    if: >-
+      contains(needs.get_commit_message.outputs.message, '1') ||
+      github.event_name == 'schedule' ||
+      github.event_name == 'workflow_dispatch'
+    runs-on: ${{ matrix.buildplat[0] }}
+    strategy:
+      # Ensure that a wheel builder finishes even if another fails
+      fail-fast: false
+      matrix:
+        # Github Actions doesn't support pairing matrix values together, let's improvise
+        # https://github.com/github/feedback/discussions/7835#discussioncomment-1769026
+        buildplat:
+          - [ubuntu-22.04, manylinux, x86_64, "", ""]
+          - [ubuntu-22.04, musllinux, x86_64, "", ""]
+          # TODO: build scipy and set up Windows and MacOS
+          # cibuildwheel does not yet support Mac for free-threaded python
+          # windows is supported but numpy doesn't build on the image yet
+        python: [["cp313t", '3.13']]
+    env:
+      IS_32_BIT: ${{ matrix.buildplat[2] == 'x86' }}
+      # upload to staging if it's a push to a maintenance branch and the last
+      # commit message contains '[wheel build]'
+      IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/heads/maintenance') && contains(needs.get_commit_message.outputs.message, '1') }}
+      IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
+    steps:
+      - name: Checkout scipy
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          submodules: true
+
+      # Used to push the built wheels
+      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+        with:
+          python-version: "3.x"
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@ba8be0d98853f5744f24e7f902c8adef7ae2e7f3  # v2.18.1
+        env:
+          CIBW_PRERELEASE_PYTHONS: True
+          CIBW_FREE_THREADED_SUPPORT: True
+          CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}*
+          CIBW_ARCHS: ${{ matrix.buildplat[2] }}
+          # TODO: remove along with installing build deps in
+          # cibw_before_build.sh when a released cython can build numpy
+          CIBW_BUILD_FRONTEND: "pip; args: --no-build-isolation"
+
+      - uses: actions/upload-artifact@v4
+        with:
+          path: ./wheelhouse/*.whl
+          name: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
+            ${{ matrix.buildplat[2] }} ${{ matrix.buildplat[3] }}
+            ${{ matrix.buildplat[4] }}
+
+      - uses: mamba-org/setup-micromamba@422500192359a097648154e8db4e39bdb6c6eed7
+        with:
+          # for installation of anaconda-client, required for upload to
+          # anaconda.org
+          # Note that this step is *after* specific pythons have been used to
+          # build and test the wheel
+          # for installation of anaconda-client, for upload to anaconda.org
+          # environment will be activated after creation, and in future bash steps
+          init-shell: bash
+          environment-name: upload-env
+          create-args: >-
+            anaconda-client
+
+      - name: Upload wheels
+        if: success()
+        shell: bash -el {0}
+        # see https://github.com/marketplace/actions/setup-miniconda for why
+        # `-el {0}` is required.
+        env:
+          SCIPY_STAGING_UPLOAD_TOKEN: ${{ secrets.SCIPY_STAGING_UPLOAD_TOKEN }}
+          SCIPY_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.SCIPY_NIGHTLY_UPLOAD_TOKEN }}
+        run: |
+          conda install -y anaconda-client
+          source tools/wheels/upload_wheels.sh
+          set_upload_vars
+          # For cron jobs (restricted to main branch) or "Run workflow" trigger
+          # an upload to:
+          #
+          # https://anaconda.org/scientific-python-nightly-wheels/scipy
+          #
+          # Pushes to a maintenance branch that contain '[wheel build]' will
+          # cause wheels to be built and uploaded to:
+          #
+          # https://anaconda.org/multibuild-wheels-staging/scipy
+          #
+          # The tokens were originally generated at anaconda.org
+          upload_wheels

.github/workflows/issue-labeler.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     # label based on issue title
-    - uses: github/issue-labeler@v3.3
+    - uses: github/issue-labeler@v3.4
       if: github.repository == 'scipy/scipy'
       with:
         configuration-path: .github/labeler.yml

.github/workflows/linux.yml

@@ -84,7 +84,7 @@ jobs:
         echo "timestamp=${NOW}" >> $GITHUB_OUTPUT
 
     - name: Setup compiler cache
-      uses:  actions/cache@v3
+      uses:  actions/cache@v4
       id:    cache-ccache
       # Reference: https://docs.github.com/en/actions/guides/caching-dependencies-to-speed-up-workflows#matching-a-cache-key
       # NOTE: The caching strategy is modeled in a way that it will always have a unique cache key for each workflow run
@@ -308,7 +308,7 @@ jobs:
         sudo apt-get install -y libgmp-dev libmpfr-dev libmpc-dev ccache gfortran
 
     - name: Caching Python dependencies
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: cache
       with:
         path: ~/.cache/pip
@@ -321,10 +321,8 @@ jobs:
         # (it can be put back after matplotlib has made a 2.0-compatible
         # release on PyPI.
         python -m pip install --pre --upgrade pytest pytest-cov pytest-xdist mpmath gmpy2 threadpoolctl pooch hypothesis
-        # TODO: once the scipy_ symbol prefix issue is fixed, install
-        # scipy-openblas32 from the pre-releases bucket again (see gh-19640)
-        python -m pip install "scipy-openblas32<=0.3.23.293.2"
-        # Install numpy last, to ensure we get 2.0.0-dev (avoid possible <2.0 constraints).
+        python -m pip install -r requirements/openblas.txt
+        # Install numpy last, to ensure we get nightly (avoid possible <2.0 constraints).
         python -m pip install --pre --upgrade --timeout=60 -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
 
     - name:  Prepare compiler cache
@@ -337,7 +335,7 @@ jobs:
         echo "timestamp=${NOW}" >> $GITHUB_OUTPUT
 
     - name: Setup compiler cache
-      uses:  actions/cache@v3
+      uses:  actions/cache@v4
       id:    cache-ccache
       with:
         path: ${{ steps.prep-ccache.outputs.dir }}
@@ -382,20 +380,27 @@ jobs:
 
     - name: build + test
       run: |
-        set -euo pipefail
+        set -exuo pipefail
         docker pull quay.io/pypa/manylinux2014_i686
         docker run -v $(pwd):/scipy --platform=linux/i386 quay.io/pypa/manylinux2014_i686 /bin/bash -c "cd /scipy && \
         uname -a && \
-        basedir=\$(python3.10 tools/openblas_support.py) && \
-        cp -r \$basedir/lib/* /usr/local/lib && \
-        cp \$basedir/include/* /usr/local/include && \
-        export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig && \
         python3.10 -m venv test && \
         source test/bin/activate && \
         python -m pip install doit click rich_click pydevtool meson ninja && \
+        python -m pip install -r requirements/openblas.txt && \
+        # Ensure that scipy-openblas is picked up by the numpy<1.26 build
+        cat > \$HOME/.numpy-site.cfg <<EOL
+        [openblas]
+        libraries = \$(python -c 'import scipy_openblas32; print(scipy_openblas32.get_library())')
+        library_dirs = \$(python -c 'import scipy_openblas32; print(scipy_openblas32.get_lib_dir())')
+        include_dirs = \$(python -c 'import scipy_openblas32; print(scipy_openblas32.get_include_dir())')
+        runtime_library_dirs = \$(python -c 'import scipy_openblas32; print(scipy_openblas32.get_lib_dir())')
+        symbol_prefix = scipy_
+        EOL
         python -m pip install numpy==1.23.5 cython pybind11 pytest pytest-timeout pytest-xdist pytest-env 'Pillow<10.0.0' mpmath pythran pooch meson hypothesis && \
-        LD_LIBRARY_PATH=/usr/local/lib python dev.py build && \
-        LD_LIBRARY_PATH=/usr/local/lib python dev.py test"
+        python -c 'import numpy as np; np.show_config()' && \
+        python dev.py build --with-scipy-openblas && \
+        python dev.py --no-build test"
 
   #################################################################################
   distro_multiple_pythons:
@@ -441,3 +446,91 @@ jobs:
           python3.11 -m pytest --pyargs scipy.cluster
           python3.11 -m pytest --pyargs scipy.linalg
           popd
+
+  free-threaded:
+    needs: get_commit_message
+    runs-on: ubuntu-latest
+    if: >
+      needs.get_commit_message.outputs.message == 1
+      && (github.repository == 'scipy/scipy' || github.repository == '')
+    steps:
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        submodules: recursive
+        fetch-tags: true
+    # TODO: replace with setup-python when there is support
+    - uses: deadsnakes/action@6c8b9b82fe0b4344f4b98f2775fcc395df45e494 # v3.1.0
+      with:
+          python-version: '3.13-dev'
+          nogil: true
+    - name: Install Ubuntu dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libgmp-dev libmpfr-dev libmpc-dev ccache gfortran
+    # TODO: remove pip pre-release install after Python 3.13 release
+    - name: Install pre-release pip
+      run: |
+        pip install -U --pre pip
+    # TODO: remove cython nightly install when cython does a release
+    - name: Install nightly Cython
+      run: |
+        pip install git+https://github.com/cython/cython
+    - name: Install nightly NumPy
+      run: |
+        pip install -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
+    - name: Install Python dependencies
+      run: |
+        pip install git+https://github.com/serge-sans-paille/pythran
+        pip install ninja meson-python pybind11 click rich_click pydevtool
+        pip install --pre --upgrade pytest pytest-xdist gmpy2 threadpoolctl pooch hypothesis
+        pip install -r requirements/openblas.txt
+    - name: Build and run tests
+      env:
+        PYTHON_GIL: 0
+      # TODO: For some reason the Meson installation path points to
+      # python3/site-packages as opposed to python3.13/site-packages,
+      # then the dev.py scripts do not work as expected.
+      run: |
+        # python dev.py build --with-scipy-openblas
+        # python dev.py --no-build test -j2 --mode full
+        python -c "import scipy_openblas32; print(scipy_openblas32.get_pkg_config())" > scipy-openblas.pc
+        PKG_CONFIG_PATH="$PWD" pip install . -vv --no-build-isolation
+        pushd $RUNNER_TEMP
+        PYTHON_GIL=0 python -m pytest --pyargs scipy -n2 --durations=10
+  #################################################################################
+  clang-17-build-only:
+    # Purpose is to check for warnings in builds with latest clang.
+    # We do not run the test suite here.
+    name: Clang-17 build-only (-Werror)
+    needs: get_commit_message
+    if: >
+      needs.get_commit_message.outputs.message == 1
+      && (github.repository == 'scipy/scipy' || github.repository == '')
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/[email protected]
+        with:
+          submodules: recursive
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Setup system dependencies
+        run: |
+          sudo apt-get -y update
+          wget https://apt.llvm.org/llvm.sh
+          chmod u+x llvm.sh
+          sudo ./llvm.sh 17
+          sudo apt install -y libopenblas-dev liblapack-dev
+
+      - name: Setup Python build deps
+        run: |
+          pip install -r requirements/build.txt
+          pip install build
+
+      - name: Build wheel, check for compiler warnings
+        run: |
+          # specify which compilers to use using environment variables
+          CC=clang-17 CXX=clang++-17 FC=gfortran python -m build -wnx -Csetup-args=--werror