gh-128563: A new tail-calling interpreter

.gitattributes

@@ -95,6 +95,7 @@ Programs/test_frozenmain.h                          generated
 Python/Python-ast.c                                 generated
 Python/executor_cases.c.h                           generated
 Python/generated_cases.c.h                          generated
+Python/generated_tail_call_handlers.c.h             generated
 Python/optimizer_cases.c.h                          generated
 Python/opcode_targets.h                             generated
 Python/stdlib_module_names.h                        generated

.github/workflows/tail-call.yml

@@ -0,0 +1,117 @@
+name: Tail calling interpreter
+on:
+  pull_request:
+    paths:
+      - 'Python/bytecodes.c'
+      - 'Python/ceval.c'
+      - 'Python/ceval_macros.h'
+      - 'Python/generated_tail_call_handlers.c.h'
+  push:
+    paths:
+      - 'Python/bytecodes.c'
+      - 'Python/ceval.c'
+      - 'Python/ceval_macros.h'
+      - 'Python/generated_tail_call_handlers.c.h'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  jit:
+    name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }})
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+# Un-comment as we add support for more platforms for tail-calling interpreters.
+#          - i686-pc-windows-msvc/msvc
+#          - x86_64-pc-windows-msvc/msvc
+#          - aarch64-pc-windows-msvc/msvc
+          - x86_64-apple-darwin/clang
+          - aarch64-apple-darwin/clang
+          - x86_64-unknown-linux-gnu/gcc
+          - aarch64-unknown-linux-gnu/gcc
+        debug:
+          - true
+          - false
+        llvm:
+          - 19
+        include:
+#          - target: i686-pc-windows-msvc/msvc
+#            architecture: Win32
+#            runner: windows-latest
+#          - target: x86_64-pc-windows-msvc/msvc
+#            architecture: x64
+#            runner: windows-latest
+#          - target: aarch64-pc-windows-msvc/msvc
+#            architecture: ARM64
+#            runner: windows-latest
+          - target: x86_64-apple-darwin/clang
+            architecture: x86_64
+            runner: macos-13
+          - target: aarch64-apple-darwin/clang
+            architecture: aarch64
+            runner: macos-14
+          - target: x86_64-unknown-linux-gnu/gcc
+            architecture: x86_64
+            runner: ubuntu-24.04
+          - target: aarch64-unknown-linux-gnu/gcc
+            architecture: aarch64
+            # Forks don't have access to our paid AArch64 runners. These jobs are skipped below:
+            runner: ${{ github.repository_owner == 'python' && 'ubuntu-24.04-aarch64' || 'ubuntu-24.04' }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Native Windows
+        if: runner.os == 'Windows' && matrix.architecture != 'ARM64'
+        run: |
+          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0
+          ./PCbuild/build.bat --tail-call-interp ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }}
+          ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+
+      # No tests (yet):
+      - name: Emulated Windows
+        if: runner.os == 'Windows' && matrix.architecture == 'ARM64'
+        run: |
+          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0
+          ./PCbuild/build.bat --tail-call-interp ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }}
+
+        # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966.
+        # This is a bug in the macOS runner image where the pre-installed Python is installed in the same
+        # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes
+        # the symlink to the pre-installed Python so that the Homebrew Python is used instead.
+      - name: Native macOS
+        if: runner.os == 'macOS'
+        run: |
+          brew update
+          find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete
+          brew install llvm@${{ matrix.llvm }}
+          export SDKROOT="$(xcrun --show-sdk-path)"
+          export PATH="/opt/homebrew/opt/llvm/bin:$PATH"
+          export PATH="/usr/local/opt/llvm/bin:$PATH"
+          CC=clang-19 ./configure --with-tail-call-interp ${{ matrix.debug && '--with-pydebug' || '' }}
+          make all --jobs 4
+          ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+
+      - name: Native Linux
+        # Forks don't have access to our paid AArch64 runners. Skip those:
+        if: runner.os == 'Linux' && (matrix.architecture == 'x86_64' || github.repository_owner == 'python')
+        run: |
+          sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
+          export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
+          CC=clang-19 ./configure --with-tail-call-interp ${{ matrix.debug && '--with-pydebug' || '' }}
+          make all --jobs 4
+          ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+

Include/internal/pycore_frame.h

@@ -75,7 +75,8 @@ typedef struct _PyInterpreterFrame {
     _PyStackRef *stackpointer;
     uint16_t return_offset;  /* Only relevant during a function call */
     char owner;
-    char visited;
+    char visited:4;
+    char is_entry_frame:4;
     /* Locals and stack */
     _PyStackRef localsplus[1];
 } _PyInterpreterFrame;
@@ -214,6 +215,8 @@ _PyFrame_Initialize(
         frame->localsplus[i] = PyStackRef_NULL;
     }
 
+    frame->is_entry_frame = 0;
+
 #ifdef Py_GIL_DISABLED
     // On GIL disabled, we walk the entire stack in GC. Since stacktop
     // is not always in sync with the real stack pointer, we have
@@ -394,6 +397,8 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int
     frame->visited = 0;
     frame->return_offset = 0;
 
+    frame->is_entry_frame = 0;
+
 #ifdef Py_GIL_DISABLED
     assert(code->co_nlocalsplus == 0);
     for (int i = 0; i < code->co_stacksize; i++) {

Lib/test/test_generated_cases.py

@@ -4,6 +4,7 @@
 import sys
 import tempfile
 import unittest
+import textwrap
 
 from io import StringIO
 from test import support
@@ -36,6 +37,7 @@ def skip_if_different_mount_drives():
     import parser
     from stack import Local, Stack
     import tier1_generator
+    import tier1_tail_call_generator
     import opcode_metadata_generator
     import optimizer_generator
 
@@ -1751,6 +1753,156 @@ def test_kill_in_wrong_order(self):
             self.run_cases_test(input, "")
 
 
+class TestGeneratedTailCallErorHandlers(unittest.TestCase):
+    def setUp(self) -> None:
+        super().setUp()
+        self.maxDiff = None
+
+        self.temp_dir = tempfile.gettempdir()
+        self.temp_input_filename = os.path.join(self.temp_dir, "input.txt")
+        self.temp_output_filename = os.path.join(self.temp_dir, "output.txt")
+        self.temp_metadata_filename = os.path.join(self.temp_dir, "metadata.txt")
+        self.temp_pymetadata_filename = os.path.join(self.temp_dir, "pymetadata.txt")
+        self.temp_executor_filename = os.path.join(self.temp_dir, "executor.txt")
+
+    def tearDown(self) -> None:
+        for filename in [
+            self.temp_input_filename,
+            self.temp_output_filename,
+            self.temp_metadata_filename,
+            self.temp_pymetadata_filename,
+            self.temp_executor_filename,
+        ]:
+            try:
+                os.remove(filename)
+            except:
+                pass
+        super().tearDown()
+
+    def run_cases_test(self, input: str, expected: str):
+        with open(self.temp_input_filename, "w+") as temp_input:
+            temp_input.write(textwrap.dedent(input))
+            temp_input.flush()
+
+        with handle_stderr():
+            tier1_tail_call_generator.generate_label_handlers_from_files(
+                self.temp_input_filename, self.temp_output_filename
+            )
+
+        with open(self.temp_output_filename) as temp_output:
+            lines = temp_output.readlines()
+            while lines and lines[0].startswith(("// ", "#", "    #", "\n")):
+                lines.pop(0)
+            while lines and lines[-1].startswith(("#", "\n")):
+                lines.pop(-1)
+        actual = "".join(lines)
+
+        self.assertEqual(actual.strip(), textwrap.dedent(expected).strip())
+
+    def test_correctly_finds_pyeval_framedefault(self):
+        input = """
+        PyObject* _Py_HOT_FUNCTION
+        _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag)
+        {
+
+        /* END_BASE_INTERPRETER */
+        }
+        """
+        output = """
+        """
+        self.run_cases_test(input, output)
+
+    def test_simple_label(self):
+        input = """
+        PyObject* _Py_HOT_FUNCTION
+        _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag)
+        {
+
+        TAIL_CALL_TARGET(error):
+            DO_THING();
+        /* END_BASE_INTERPRETER */
+        }
+        """
+        output = """
+        Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS);
+
+        Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS)
+        {
+            DO_THING();
+        /* END_BASE_INTERPRETER */
+        }
+        """
+        self.run_cases_test(input, output)
+
+    def test_fallthrough_label(self):
+        input = """
+        PyObject* _Py_HOT_FUNCTION
+        _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag)
+        {
+
+        TAIL_CALL_TARGET(error):
+            DO_THING();
+        TAIL_CALL_TARGET(fallthrough):
+            DO_THING2();
+        /* END_BASE_INTERPRETER */
+        }
+        """
+        output = """
+        Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS);
+        Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_fallthrough(TAIL_CALL_PARAMS);
+
+        Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS)
+        {
+            DO_THING();
+            TAIL_CALL(fallthrough);
+        }
+
+        Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_fallthrough(TAIL_CALL_PARAMS)
+        {
+            DO_THING2();
+        /* END_BASE_INTERPRETER */
+        }
+        """
+        self.run_cases_test(input, output)
+
+    def test_transform_gotos(self):
+        input = """
+        PyObject* _Py_HOT_FUNCTION
+        _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag)
+        {
+
+        TAIL_CALL_TARGET(error):
+            if (thing) {
+                goto fallthrough;
+            }
+            DO_THING();
+        TAIL_CALL_TARGET(fallthrough):
+            DO_THING2();
+        /* END_BASE_INTERPRETER */
+        }
+        """
+        output = """
+        Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS);
+        Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_fallthrough(TAIL_CALL_PARAMS);
+
+        Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS)
+        {
+            if (thing) {
+            TAIL_CALL(fallthrough);
+            }
+            DO_THING();
+            TAIL_CALL(fallthrough);
+        }
+
+        Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_fallthrough(TAIL_CALL_PARAMS)
+        {
+            DO_THING2();
+        /* END_BASE_INTERPRETER */
+        }
+        """
+        self.run_cases_test(input, output)
+
+
 class TestGeneratedAbstractCases(unittest.TestCase):
     def setUp(self) -> None:
         super().setUp()

Makefile.pre.in

@@ -1985,7 +1985,7 @@ Objects/mimalloc/page.o: $(srcdir)/Objects/mimalloc/page-queue.c
 .PHONY: regen-cases
 regen-cases: \
         regen-opcode-ids regen-opcode-targets regen-uop-ids regen-opcode-metadata-py \
-		regen-generated-cases regen-executor-cases regen-optimizer-cases \
+		regen-generated-cases regen-generated-tail-call-handlers regen-executor-cases regen-optimizer-cases \
 		regen-opcode-metadata regen-uop-metadata
 
 .PHONY: regen-opcode-ids
@@ -2018,6 +2018,12 @@ regen-generated-cases:
 	    -o $(srcdir)/Python/generated_cases.c.h.new $(srcdir)/Python/bytecodes.c
 	$(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new
 
+.PHONY: regen-generated-tail-call-handlers
+regen-generated-tail-call-handlers:
+	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier1_tail_call_generator.py \
+	    -o $(srcdir)/Python/generated_tail_call_handlers.c.h.new $(srcdir)/Python/bytecodes.c
+	$(UPDATE_FILE) $(srcdir)/Python/generated_tail_call_handlers.c.h $(srcdir)/Python/generated_tail_call_handlers.c.h.new
+
 .PHONY: regen-executor-cases
 regen-executor-cases:
 	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier2_generator.py \
@@ -2055,6 +2061,7 @@ Python/ceval.o: \
 		$(srcdir)/Python/ceval_macros.h \
 		$(srcdir)/Python/condvar.h \
 		$(srcdir)/Python/generated_cases.c.h \
+		$(srcdir)/Python/generated_tail_call_handlers.c.h \
 		$(srcdir)/Python/executor_cases.c.h \
 		$(srcdir)/Python/opcode_targets.h
 

Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst

@@ -0,0 +1 @@
+A new type of interpreter has been added to CPython. This interpreter uses tail calls for its instruction handlers. Preliminary benchmark results suggest 7-14% geometric mean faster on pyperformance (depending on platform), and up to 45% faster on Python-intensive workloads. This interpreter currently only works on newer compilers, such as ``clang-19``. Other compilers will continue using the old interpreter. Patch by Ken Jin, with ideas by Garret Gu, Haoran Xu, and Josh Haberman.