Merge branch 'main' into pythongh-77609-glob-follow-symlinks

Author: barneygale

Diff for: .github/dependabot.yml

@@ -12,3 +12,10 @@ updates:
         update-types:
           - "version-update:semver-minor"
           - "version-update:semver-patch"
+  - package-ecosystem: "pip"
+    directory: "/Tools/clinic/"
+    schedule:
+      interval: "monthly"
+    labels:
+      - "skip issue"
+      - "skip news"

Diff for: .github/workflows/build.yml

@@ -36,6 +36,7 @@ jobs:
     timeout-minutes: 10
     outputs:
       run_tests: ${{ steps.check.outputs.run_tests }}
+      run_hypothesis: ${{ steps.check.outputs.run_hypothesis }}
     steps:
       - uses: actions/checkout@v3
       - name: Check for source changes
@@ -61,6 +62,17 @@ jobs:
             git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true
           fi
 
+          # Check if we should run hypothesis tests
+          GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}
+          echo $GIT_BRANCH
+          if $(echo "$GIT_BRANCH" | grep -q -w '3\.\(8\|9\|10\|11\)'); then
+            echo "Branch too old for hypothesis tests"
+            echo "run_hypothesis=false" >> $GITHUB_OUTPUT
+          else
+            echo "Run hypothesis tests"
+            echo "run_hypothesis=true" >> $GITHUB_OUTPUT
+          fi
+
   check_generated_files:
     name: 'Check if generated files are up to date'
     runs-on: ubuntu-latest
@@ -291,6 +303,92 @@ jobs:
     - name: SSL tests
       run: ./python Lib/test/ssltests.py
 
+  test_hypothesis:
+    name: "Hypothesis Tests on Ubuntu"
+    runs-on: ubuntu-20.04
+    timeout-minutes: 60
+    needs: check_source
+    if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true'
+    env:
+      OPENSSL_VER: 1.1.1t
+      PYTHONSTRICTEXTENSIONBUILD: 1
+    steps:
+    - uses: actions/checkout@v3
+    - name: Register gcc problem matcher
+      run: echo "::add-matcher::.github/problem-matchers/gcc.json"
+    - name: Install Dependencies
+      run: sudo ./.github/workflows/posix-deps-apt.sh
+    - name: Configure OpenSSL env vars
+      run: |
+        echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV
+        echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV
+        echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
+    - name: 'Restore OpenSSL build'
+      id: cache-openssl
+      uses: actions/cache@v3
+      with:
+        path: ./multissl/openssl/${{ env.OPENSSL_VER }}
+        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
+    - name: Install OpenSSL
+      if: steps.cache-openssl.outputs.cache-hit != 'true'
+      run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux
+    - name: Add ccache to PATH
+      run: |
+        echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
+    - name: Configure ccache action
+      uses: hendrikmuhs/[email protected]
+    - name: Setup directory envs for out-of-tree builds
+      run: |
+        echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV
+        echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV
+    - name: Create directories for read-only out-of-tree builds
+      run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
+    - name: Bind mount sources read-only
+      run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
+    - name: Configure CPython out-of-tree
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: ../cpython-ro-srcdir/configure --with-pydebug --with-openssl=$OPENSSL_DIR
+    - name: Build CPython out-of-tree
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: make -j4
+    - name: Display build info
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: make pythoninfo
+    - name: Remount sources writable for tests
+      # some tests write to srcdir, lack of pyc files slows down testing
+      run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
+    - name: Setup directory envs for out-of-tree builds
+      run: |
+        echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV
+    - name: "Create hypothesis venv"
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: |
+        VENV_LOC=$(realpath -m .)/hypovenv
+        VENV_PYTHON=$VENV_LOC/bin/python
+        echo "HYPOVENV=${VENV_LOC}" >> $GITHUB_ENV
+        echo "VENV_PYTHON=${VENV_PYTHON}" >> $GITHUB_ENV
+        ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -U hypothesis
+    - name: "Run tests"
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: |
+        # Most of the excluded tests are slow test suites with no property tests
+        #
+        # (GH-104097) test_sysconfig is skipped because it has tests that are
+        # failing when executed from inside a virtual environment.
+        ${{ env.VENV_PYTHON }} -m test \
+          -W \
+          -o \
+          -j4 \
+          -x test_asyncio \
+          -x test_multiprocessing_fork \
+          -x test_multiprocessing_forkserver \
+          -x test_multiprocessing_spawn \
+          -x test_concurrent_futures \
+          -x test_socket \
+          -x test_subprocess \
+          -x test_signal \
+          -x test_sysconfig
+
 
   build_asan:
     name: 'Address sanitizer'

Diff for: .github/workflows/mypy.yml

@@ -0,0 +1,39 @@
+# Workflow to run mypy on select parts of the CPython repo
+name: mypy
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - "Tools/clinic/**"
+      - ".github/workflows/mypy.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+env:
+  PIP_DISABLE_PIP_VERSION_CHECK: 1
+  FORCE_COLOR: 1
+  TERM: xterm-256color  # needed for FORCE_COLOR to work on mypy on Ubuntu, see https://github.com/python/mypy/issues/13817
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  mypy:
+    name: Run mypy on Tools/clinic/
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.x"
+          cache: pip
+          cache-dependency-path: Tools/clinic/requirements-dev.txt
+      - run: pip install -r Tools/clinic/requirements-dev.txt
+      - run: mypy --config-file Tools/clinic/mypy.ini

Diff for: Doc/howto/clinic.rst

@@ -775,6 +775,9 @@ All Argument Clinic converters accept the following arguments:
     because :pep:`8` mandates that the Python library may not use
     annotations.
 
+  ``unused``
+    Wrap the argument with :c:macro:`Py_UNUSED` in the impl function signature.
+
 In addition, some converters accept additional arguments.  Here is a list
 of these arguments, along with their meanings:
 

Diff for: Doc/library/atexit.rst

@@ -20,6 +20,9 @@ at interpreter termination time they will be run in the order ``C``, ``B``,
 program is killed by a signal not handled by Python, when a Python fatal
 internal error is detected, or when :func:`os._exit` is called.
 
+**Note:** The effect of registering or unregistering functions from within
+a cleanup function is undefined.
+
 .. versionchanged:: 3.7
     When used with C-API subinterpreters, registered functions
     are local to the interpreter they were registered in.

Diff for: Doc/library/functions.rst

@@ -168,6 +168,13 @@ are always available.  They are listed here in alphabetical order.
    If :func:`sys.breakpointhook` is not accessible, this function will
    raise :exc:`RuntimeError`.
 
+   By default, the behavior of :func:`breakpoint` can be changed with
+   the :envvar:`PYTHONBREAKPOINT` environment variable.
+   See :func:`sys.breakpointhook` for usage details.
+
+   Note that this is not guaranteed if :func:`sys.breakpointhook`
+   has been replaced.
+
    .. audit-event:: builtins.breakpoint breakpointhook breakpoint
 
    .. versionadded:: 3.7

Diff for: Doc/library/pdb.rst

@@ -602,9 +602,17 @@ can be overridden by the local file.
 
    Execute the (one-line) *statement* in the context of the current stack frame.
    The exclamation point can be omitted unless the first word of the statement
-   resembles a debugger command.  To set a global variable, you can prefix the
-   assignment command with a :keyword:`global` statement on the same line,
-   e.g.::
+   resembles a debugger command, e.g.:
+
+   .. code-block:: none
+
+      (Pdb) ! n=42
+      (Pdb)
+
+   To set a global variable, you can prefix the assignment command with a
+   :keyword:`global` statement on the same line, e.g.:
+
+   .. code-block:: none
 
       (Pdb) global list_options; list_options = ['-l']
       (Pdb)

Diff for: Doc/library/random.rst

@@ -334,8 +334,10 @@ be found in any statistics text.
 
 .. function:: gammavariate(alpha, beta)
 
-   Gamma distribution.  (*Not* the gamma function!)  Conditions on the
-   parameters are ``alpha > 0`` and ``beta > 0``.
+   Gamma distribution.  (*Not* the gamma function!)  The shape and
+   scale parameters, *alpha* and *beta*, must have positive values.
+   (Calling conventions vary and some sources define 'beta'
+   as the inverse of the scale).
 
    The probability distribution function is::
 
@@ -346,7 +348,8 @@ be found in any statistics text.
 
 .. function:: gauss(mu=0.0, sigma=1.0)
 
-   Normal distribution, also called the Gaussian distribution.  *mu* is the mean,
+   Normal distribution, also called the Gaussian distribution.
+   *mu* is the mean,
    and *sigma* is the standard deviation.  This is slightly faster than
    the :func:`normalvariate` function defined below.
 

Diff for: Doc/library/sqlite3.rst

@@ -1694,7 +1694,10 @@ Cursor objects
       ``INSERT``, ``UPDATE``, ``DELETE``, and ``REPLACE`` statements;
       is ``-1`` for other statements,
       including :abbr:`CTE (Common Table Expression)` queries.
-      It is only updated by the :meth:`execute` and :meth:`executemany` methods.
+      It is only updated by the :meth:`execute` and :meth:`executemany` methods,
+      after the statement has run to completion.
+      This means that any resulting rows must be fetched in order for
+      :attr:`!rowcount` to be updated.
 
    .. attribute:: row_factory
 

Diff for: Doc/library/typing.rst

@@ -19,7 +19,7 @@
 
 This module provides runtime support for type hints. The most fundamental
 support consists of the types :data:`Any`, :data:`Union`, :data:`Callable`,
-:class:`TypeVar`, and :class:`Generic`. For a full specification, please see
+:class:`TypeVar`, and :class:`Generic`. For a specification, please see
 :pep:`484`. For a simplified introduction to type hints, see :pep:`483`.
 
 
@@ -592,7 +592,7 @@ The module defines the following classes, functions and decorators.
    when the checked program targets Python 3.9 or newer.
 
    The deprecated types will be removed from the :mod:`typing` module
-   in the first Python version released 5 years after the release of Python 3.9.0.
+   no sooner than the first Python version released 5 years after the release of Python 3.9.0.
    See details in :pep:`585`—*Type Hinting Generics In Standard Collections*.
 
 
@@ -1291,6 +1291,8 @@ These are not used in annotations. They are building blocks for creating generic
        U = TypeVar('U', bound=str|bytes)  # Can be any subtype of the union str|bytes
        V = TypeVar('V', bound=SupportsAbs)  # Can be anything with an __abs__ method
 
+.. _typing-constrained-typevar:
+
     Using a *constrained* type variable, however, means that the ``TypeVar``
     can only ever be solved as being exactly one of the constraints given::
 
@@ -1550,7 +1552,7 @@ These are not used in annotations. They are building blocks for creating generic
 
 .. data:: AnyStr
 
-   ``AnyStr`` is a :class:`constrained type variable <TypeVar>` defined as
+   ``AnyStr`` is a :ref:`constrained type variable <typing-constrained-typevar>` defined as
    ``AnyStr = TypeVar('AnyStr', str, bytes)``.
 
    It is meant to be used for functions that may accept any kind of string
@@ -2112,7 +2114,7 @@ Other concrete types
       Python 2 is no longer supported, and most type checkers also no longer
       support type checking Python 2 code. Removal of the alias is not
       currently planned, but users are encouraged to use
-      :class:`str` instead of ``Text`` wherever possible.
+      :class:`str` instead of ``Text``.
 
 Abstract Base Classes
 ---------------------

Diff for: Doc/whatsnew/3.12.rst

@@ -831,6 +831,9 @@ Pending Removal in Python 3.14
   For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`.
   (Contributed by Shantanu Jain in :gh:`91896`.)
 
+* :class:`typing.ByteString`, deprecated since Python 3.9, now causes a
+  :exc:`DeprecationWarning` to be emitted when it is used.
+
 * Creating immutable types (:data:`Py_TPFLAGS_IMMUTABLETYPE`) with mutable
   bases using the C API.
 
@@ -1201,6 +1204,8 @@ Build Changes
 
   (Contributed by Zhang Na in :gh:`90656`.)
 
+* ``PYTHON_FOR_REGEN`` now require Python 3.10 or newer.
+
 
 C API Changes
 =============

Diff for: Include/internal/pycore_code.h

@@ -53,9 +53,6 @@ typedef struct {
 
 typedef struct {
     uint16_t counter;
-    uint16_t class_version[2];
-    uint16_t self_type_version[2];
-    uint16_t method[4];
 } _PySuperAttrCache;
 
 #define INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR CACHE_ENTRIES(_PySuperAttrCache)
@@ -227,8 +224,8 @@ extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range);
 
 /* Specialization functions */
 
-extern void _Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls, PyObject *self,
-                                         _Py_CODEUNIT *instr, PyObject *name, int load_method);
+extern void _Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls,
+                                         _Py_CODEUNIT *instr, int load_method);
 extern void _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr,
                                     PyObject *name);
 extern void _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr,

Diff for: Include/internal/pycore_frame.h

@@ -19,7 +19,6 @@ struct _frame {
     struct _PyInterpreterFrame *f_frame; /* points to the frame data */
     PyObject *f_trace;          /* Trace function */
     int f_lineno;               /* Current line number. Only valid if non-zero */
-    int f_last_traced_line;     /* The last line traced for this frame */
     char f_trace_lines;         /* Emit per-line trace events? */
     char f_trace_opcodes;       /* Emit per-opcode trace events? */
     char f_fast_as_locals;      /* Have the fast locals of this frame been converted to a dict? */

Diff for: Include/internal/pycore_instruments.h

@@ -69,13 +69,13 @@ _Py_call_instrumentation(PyThreadState *tstate, int event,
 
 extern int
 _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame,
-                              _Py_CODEUNIT *instr);
+                              _Py_CODEUNIT *instr, _Py_CODEUNIT *prev);
 
 extern int
 _Py_call_instrumentation_instruction(
     PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr);
 
-int
+_Py_CODEUNIT *
 _Py_call_instrumentation_jump(
     PyThreadState *tstate, int event,
     _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target);
@@ -100,6 +100,7 @@ extern int
 _Py_Instrumentation_GetLine(PyCodeObject *code, int index);
 
 extern PyObject _PyInstrumentation_MISSING;
+extern PyObject _PyInstrumentation_DISABLE;
 
 #ifdef __cplusplus
 }