From 5f974fa39f1ffb3e36f7b875ec52d2f64c2a8a99 Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Sun, 10 Aug 2025 00:06:16 +0200 Subject: [PATCH 1/3] python313: revert commit that locks ssl sockets This breaks among other things the websockets package and is due to be reverted in 3.13.7. --- .../python/cpython/3.13/revert-gh134724.patch | 373 ++++++++++++++++++ .../interpreters/python/cpython/default.nix | 4 + 2 files changed, 377 insertions(+) create mode 100644 pkgs/development/interpreters/python/cpython/3.13/revert-gh134724.patch diff --git a/pkgs/development/interpreters/python/cpython/3.13/revert-gh134724.patch b/pkgs/development/interpreters/python/cpython/3.13/revert-gh134724.patch new file mode 100644 index 0000000000000..f48dcf3b39da9 --- /dev/null +++ b/pkgs/development/interpreters/python/cpython/3.13/revert-gh134724.patch @@ -0,0 +1,373 @@ +commit 4a37dd6cef1556c64c2665061b5e01bbd2bb3a82 +Author: Gregory P. Smith <68491+gpshead@users.noreply.github.com> +Date: Sun Jul 27 08:30:25 2025 -0700 + + [3.13] gh-134698: Hold a lock when the thread state is detached in ssl (GH-134724) (#137126) + + Lock when the thread state is detached. + (cherry picked from commit e047a35b23c1aa69ab8d5da56f36319cec4d36b8) or really from the 3.14 backport fd565fdfc9c0001900d03d627e2fda83f1bcca90 + + Co-authored-by: Peter Bierma + +diff --git b/Modules/_ssl.c a/Modules/_ssl.c +index 981c3d6a936..aa846f68641 100644 +--- b/Modules/_ssl.c ++++ a/Modules/_ssl.c +@@ -42,14 +42,14 @@ + /* Redefined below for Windows debug builds after important #includes */ + #define _PySSL_FIX_ERRNO + +-#define PySSL_BEGIN_ALLOW_THREADS_S(save, mutex) \ +- do { (save) = PyEval_SaveThread(); PyMutex_Lock(mutex); } while(0) +-#define PySSL_END_ALLOW_THREADS_S(save, mutex) \ +- do { PyMutex_Unlock(mutex); PyEval_RestoreThread(save); _PySSL_FIX_ERRNO; } while(0) +-#define PySSL_BEGIN_ALLOW_THREADS(self) { \ ++#define PySSL_BEGIN_ALLOW_THREADS_S(save) \ ++ do { (save) = PyEval_SaveThread(); } while(0) ++#define PySSL_END_ALLOW_THREADS_S(save) \ ++ do { PyEval_RestoreThread(save); _PySSL_FIX_ERRNO; } while(0) ++#define PySSL_BEGIN_ALLOW_THREADS { \ + PyThreadState *_save = NULL; \ +- PySSL_BEGIN_ALLOW_THREADS_S(_save, &self->tstate_mutex); +-#define PySSL_END_ALLOW_THREADS(self) PySSL_END_ALLOW_THREADS_S(_save, &self->tstate_mutex); } ++ PySSL_BEGIN_ALLOW_THREADS_S(_save); ++#define PySSL_END_ALLOW_THREADS PySSL_END_ALLOW_THREADS_S(_save); } + + #if defined(HAVE_POLL_H) + #include +@@ -304,9 +304,6 @@ typedef struct { + PyObject *psk_client_callback; + PyObject *psk_server_callback; + #endif +- /* Lock to synchronize calls when the thread state is detached. +- See also gh-134698. */ +- PyMutex tstate_mutex; + } PySSLContext; + + typedef struct { +@@ -332,9 +329,6 @@ typedef struct { + * and shutdown methods check for chained exceptions. + */ + PyObject *exc; +- /* Lock to synchronize calls when the thread state is detached. +- See also gh-134698. */ +- PyMutex tstate_mutex; + } PySSLSocket; + + typedef struct { +@@ -846,14 +840,13 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, + self->server_hostname = NULL; + self->err = err; + self->exc = NULL; +- self->tstate_mutex = (PyMutex){0}; + + /* Make sure the SSL error state is initialized */ + ERR_clear_error(); + +- PySSL_BEGIN_ALLOW_THREADS(sslctx) ++ PySSL_BEGIN_ALLOW_THREADS + self->ssl = SSL_new(ctx); +- PySSL_END_ALLOW_THREADS(sslctx) ++ PySSL_END_ALLOW_THREADS + if (self->ssl == NULL) { + Py_DECREF(self); + _setSSLError(get_state_ctx(self), NULL, 0, __FILE__, __LINE__); +@@ -919,12 +912,12 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, + BIO_set_nbio(SSL_get_wbio(self->ssl), 1); + } + +- PySSL_BEGIN_ALLOW_THREADS(self) ++ PySSL_BEGIN_ALLOW_THREADS + if (socket_type == PY_SSL_CLIENT) + SSL_set_connect_state(self->ssl); + else + SSL_set_accept_state(self->ssl); +- PySSL_END_ALLOW_THREADS(self) ++ PySSL_END_ALLOW_THREADS + + self->socket_type = socket_type; + if (sock != NULL) { +@@ -993,10 +986,10 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) + /* Actually negotiate SSL connection */ + /* XXX If SSL_do_handshake() returns 0, it's also a failure. */ + do { +- PySSL_BEGIN_ALLOW_THREADS(self) ++ PySSL_BEGIN_ALLOW_THREADS + ret = SSL_do_handshake(self->ssl); + err = _PySSL_errno(ret < 1, self->ssl, ret); +- PySSL_END_ALLOW_THREADS(self) ++ PySSL_END_ALLOW_THREADS + self->err = err; + + if (PyErr_CheckSignals()) +@@ -2369,10 +2362,9 @@ PySSL_select(PySocketSockObject *s, int writing, PyTime_t timeout) + ms = (int)_PyTime_AsMilliseconds(timeout, _PyTime_ROUND_CEILING); + assert(ms <= INT_MAX); + +- Py_BEGIN_ALLOW_THREADS ++ PySSL_BEGIN_ALLOW_THREADS + rc = poll(&pollfd, 1, (int)ms); +- Py_END_ALLOW_THREADS +- _PySSL_FIX_ERRNO; ++ PySSL_END_ALLOW_THREADS + #else + /* Guard against socket too large for select*/ + if (!_PyIsSelectable_fd(s->sock_fd)) +@@ -2384,14 +2376,13 @@ PySSL_select(PySocketSockObject *s, int writing, PyTime_t timeout) + FD_SET(s->sock_fd, &fds); + + /* Wait until the socket becomes ready */ +- Py_BEGIN_ALLOW_THREADS ++ PySSL_BEGIN_ALLOW_THREADS + nfds = Py_SAFE_DOWNCAST(s->sock_fd+1, SOCKET_T, int); + if (writing) + rc = select(nfds, NULL, &fds, NULL, &tv); + else + rc = select(nfds, &fds, NULL, NULL, &tv); +- Py_END_ALLOW_THREADS +- _PySSL_FIX_ERRNO; ++ PySSL_END_ALLOW_THREADS + #endif + + /* Return SOCKET_TIMED_OUT on timeout, SOCKET_OPERATION_OK otherwise +@@ -2462,10 +2453,10 @@ _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b) + } + + do { +- PySSL_BEGIN_ALLOW_THREADS(self) ++ PySSL_BEGIN_ALLOW_THREADS + retval = SSL_write_ex(self->ssl, b->buf, (size_t)b->len, &count); + err = _PySSL_errno(retval == 0, self->ssl, retval); +- PySSL_END_ALLOW_THREADS(self) ++ PySSL_END_ALLOW_THREADS + self->err = err; + + if (PyErr_CheckSignals()) +@@ -2523,10 +2514,10 @@ _ssl__SSLSocket_pending_impl(PySSLSocket *self) + int count = 0; + _PySSLError err; + +- PySSL_BEGIN_ALLOW_THREADS(self) ++ PySSL_BEGIN_ALLOW_THREADS + count = SSL_pending(self->ssl); + err = _PySSL_errno(count < 0, self->ssl, count); +- PySSL_END_ALLOW_THREADS(self) ++ PySSL_END_ALLOW_THREADS + self->err = err; + + if (count < 0) +@@ -2617,10 +2608,10 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, + deadline = _PyDeadline_Init(timeout); + + do { +- PySSL_BEGIN_ALLOW_THREADS(self) ++ PySSL_BEGIN_ALLOW_THREADS + retval = SSL_read_ex(self->ssl, mem, (size_t)len, &count); + err = _PySSL_errno(retval == 0, self->ssl, retval); +- PySSL_END_ALLOW_THREADS(self) ++ PySSL_END_ALLOW_THREADS + self->err = err; + + if (PyErr_CheckSignals()) +@@ -2719,7 +2710,7 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) + } + + while (1) { +- PySSL_BEGIN_ALLOW_THREADS(self) ++ PySSL_BEGIN_ALLOW_THREADS + /* Disable read-ahead so that unwrap can work correctly. + * Otherwise OpenSSL might read in too much data, + * eating clear text data that happens to be +@@ -2732,7 +2723,7 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) + SSL_set_read_ahead(self->ssl, 0); + ret = SSL_shutdown(self->ssl); + err = _PySSL_errno(ret < 0, self->ssl, ret); +- PySSL_END_ALLOW_THREADS(self) ++ PySSL_END_ALLOW_THREADS + self->err = err; + + /* If err == 1, a secure shutdown with SSL_shutdown() is complete */ +@@ -3124,10 +3115,9 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) + // no other thread can be touching this object yet. + // (Technically, we can't even lock if we wanted to, as the + // lock hasn't been initialized yet.) +- Py_BEGIN_ALLOW_THREADS ++ PySSL_BEGIN_ALLOW_THREADS + ctx = SSL_CTX_new(method); +- Py_END_ALLOW_THREADS +- _PySSL_FIX_ERRNO; ++ PySSL_END_ALLOW_THREADS + + if (ctx == NULL) { + _setSSLError(get_ssl_state(module), NULL, 0, __FILE__, __LINE__); +@@ -3153,7 +3143,6 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) + self->psk_client_callback = NULL; + self->psk_server_callback = NULL; + #endif +- self->tstate_mutex = (PyMutex){0}; + + /* Don't check host name by default */ + if (proto_version == PY_SSL_VERSION_TLS_CLIENT) { +@@ -3270,10 +3259,9 @@ context_clear(PySSLContext *self) + Py_CLEAR(self->psk_server_callback); + #endif + if (self->keylog_bio != NULL) { +- Py_BEGIN_ALLOW_THREADS ++ PySSL_BEGIN_ALLOW_THREADS + BIO_free_all(self->keylog_bio); +- Py_END_ALLOW_THREADS +- _PySSL_FIX_ERRNO; ++ PySSL_END_ALLOW_THREADS + self->keylog_bio = NULL; + } + return 0; +@@ -3992,8 +3980,7 @@ _password_callback(char *buf, int size, int rwflag, void *userdata) + _PySSLPasswordInfo *pw_info = (_PySSLPasswordInfo*) userdata; + PyObject *fn_ret = NULL; + +- pw_info->thread_state = PyThreadState_Swap(pw_info->thread_state); +- _PySSL_FIX_ERRNO; ++ PySSL_END_ALLOW_THREADS_S(pw_info->thread_state); + + if (pw_info->error) { + /* already failed previously. OpenSSL 3.0.0-alpha14 invokes the +@@ -4023,13 +4010,13 @@ _password_callback(char *buf, int size, int rwflag, void *userdata) + goto error; + } + +- pw_info->thread_state = PyThreadState_Swap(pw_info->thread_state); ++ PySSL_BEGIN_ALLOW_THREADS_S(pw_info->thread_state); + memcpy(buf, pw_info->password, pw_info->size); + return pw_info->size; + + error: + Py_XDECREF(fn_ret); +- pw_info->thread_state = PyThreadState_Swap(pw_info->thread_state); ++ PySSL_BEGIN_ALLOW_THREADS_S(pw_info->thread_state); + pw_info->error = 1; + return -1; + } +@@ -4082,10 +4069,10 @@ _ssl__SSLContext_load_cert_chain_impl(PySSLContext *self, PyObject *certfile, + SSL_CTX_set_default_passwd_cb(self->ctx, _password_callback); + SSL_CTX_set_default_passwd_cb_userdata(self->ctx, &pw_info); + } +- PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state, &self->tstate_mutex); ++ PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state); + r = SSL_CTX_use_certificate_chain_file(self->ctx, + PyBytes_AS_STRING(certfile_bytes)); +- PySSL_END_ALLOW_THREADS_S(pw_info.thread_state, &self->tstate_mutex); ++ PySSL_END_ALLOW_THREADS_S(pw_info.thread_state); + if (r != 1) { + if (pw_info.error) { + ERR_clear_error(); +@@ -4100,11 +4087,11 @@ _ssl__SSLContext_load_cert_chain_impl(PySSLContext *self, PyObject *certfile, + } + goto error; + } +- PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state, &self->tstate_mutex); ++ PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state); + r = SSL_CTX_use_PrivateKey_file(self->ctx, + PyBytes_AS_STRING(keyfile ? keyfile_bytes : certfile_bytes), + SSL_FILETYPE_PEM); +- PySSL_END_ALLOW_THREADS_S(pw_info.thread_state, &self->tstate_mutex); ++ PySSL_END_ALLOW_THREADS_S(pw_info.thread_state); + Py_CLEAR(keyfile_bytes); + Py_CLEAR(certfile_bytes); + if (r != 1) { +@@ -4121,9 +4108,9 @@ _ssl__SSLContext_load_cert_chain_impl(PySSLContext *self, PyObject *certfile, + } + goto error; + } +- PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state, &self->tstate_mutex); ++ PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state); + r = SSL_CTX_check_private_key(self->ctx); +- PySSL_END_ALLOW_THREADS_S(pw_info.thread_state, &self->tstate_mutex); ++ PySSL_END_ALLOW_THREADS_S(pw_info.thread_state); + if (r != 1) { + _setSSLError(get_state_ctx(self), NULL, 0, __FILE__, __LINE__); + goto error; +@@ -4340,9 +4327,9 @@ _ssl__SSLContext_load_verify_locations_impl(PySSLContext *self, + cafile_buf = PyBytes_AS_STRING(cafile_bytes); + if (capath) + capath_buf = PyBytes_AS_STRING(capath_bytes); +- PySSL_BEGIN_ALLOW_THREADS(self) ++ PySSL_BEGIN_ALLOW_THREADS + r = SSL_CTX_load_verify_locations(self->ctx, cafile_buf, capath_buf); +- PySSL_END_ALLOW_THREADS(self) ++ PySSL_END_ALLOW_THREADS + if (r != 1) { + if (errno != 0) { + PyErr_SetFromErrno(PyExc_OSError); +@@ -4394,11 +4381,10 @@ _ssl__SSLContext_load_dh_params_impl(PySSLContext *self, PyObject *filepath) + return NULL; + + errno = 0; +- Py_BEGIN_ALLOW_THREADS ++ PySSL_BEGIN_ALLOW_THREADS + dh = PEM_read_DHparams(f, NULL, NULL, NULL); + fclose(f); +- Py_END_ALLOW_THREADS +- _PySSL_FIX_ERRNO; ++ PySSL_END_ALLOW_THREADS + if (dh == NULL) { + if (errno != 0) { + PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, filepath); +@@ -4550,7 +4536,6 @@ _ssl__SSLContext_set_default_verify_paths_impl(PySSLContext *self) + Py_BEGIN_ALLOW_THREADS + rc = SSL_CTX_set_default_verify_paths(self->ctx); + Py_END_ALLOW_THREADS +- _PySSL_FIX_ERRNO; + if (!rc) { + _setSSLError(get_state_ctx(self), NULL, 0, __FILE__, __LINE__); + return NULL; +diff --git b/Modules/_ssl/debughelpers.c a/Modules/_ssl/debughelpers.c +index fb8ae7c4e0b..5fc69a07184 100644 +--- b/Modules/_ssl/debughelpers.c ++++ a/Modules/_ssl/debughelpers.c +@@ -135,15 +135,13 @@ _PySSL_keylog_callback(const SSL *ssl, const char *line) + * critical debug helper. + */ + +- assert(PyMutex_IsLocked(&ssl_obj->tstate_mutex)); +- Py_BEGIN_ALLOW_THREADS ++ PySSL_BEGIN_ALLOW_THREADS + PyThread_acquire_lock(lock, 1); + res = BIO_printf(ssl_obj->ctx->keylog_bio, "%s\n", line); + e = errno; + (void)BIO_flush(ssl_obj->ctx->keylog_bio); + PyThread_release_lock(lock); +- Py_END_ALLOW_THREADS +- _PySSL_FIX_ERRNO; ++ PySSL_END_ALLOW_THREADS + + if (res == -1) { + errno = e; +@@ -179,10 +177,9 @@ _PySSLContext_set_keylog_filename(PySSLContext *self, PyObject *arg, void *c) { + if (self->keylog_bio != NULL) { + BIO *bio = self->keylog_bio; + self->keylog_bio = NULL; +- Py_BEGIN_ALLOW_THREADS ++ PySSL_BEGIN_ALLOW_THREADS + BIO_free_all(bio); +- Py_END_ALLOW_THREADS +- _PySSL_FIX_ERRNO; ++ PySSL_END_ALLOW_THREADS + } + + if (arg == Py_None) { +@@ -204,13 +201,13 @@ _PySSLContext_set_keylog_filename(PySSLContext *self, PyObject *arg, void *c) { + self->keylog_filename = Py_NewRef(arg); + + /* Write a header for seekable, empty files (this excludes pipes). */ +- PySSL_BEGIN_ALLOW_THREADS(self) ++ PySSL_BEGIN_ALLOW_THREADS + if (BIO_tell(self->keylog_bio) == 0) { + BIO_puts(self->keylog_bio, + "# TLS secrets log file, generated by OpenSSL / Python\n"); + (void)BIO_flush(self->keylog_bio); + } +- PySSL_END_ALLOW_THREADS(self) ++ PySSL_END_ALLOW_THREADS + SSL_CTX_set_keylog_callback(self->ctx, _PySSL_keylog_callback); + return 0; + } diff --git a/pkgs/development/interpreters/python/cpython/default.nix b/pkgs/development/interpreters/python/cpython/default.nix index abdf0a1ceb228..3e1d6cc63ff19 100644 --- a/pkgs/development/interpreters/python/cpython/default.nix +++ b/pkgs/development/interpreters/python/cpython/default.nix @@ -365,6 +365,10 @@ stdenv.mkDerivation (finalAttrs: { ++ optionals (pythonAtLeast "3.13") [ ./3.13/virtualenv-permissions.patch ] + ++ optionals isPy313 [ + # https://github.com/python/cpython/issues/137583 + ./3.13/revert-gh134724.patch + ] ++ optionals mimetypesSupport [ # Make the mimetypes module refer to the right file ./mimetypes.patch From 9f8df5b2a696076b8e0e55baf8e955458f7569f5 Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Sun, 10 Aug 2025 00:07:00 +0200 Subject: [PATCH 2/3] Revert "python313Packages.websockets: temporarily disable tests" This reverts commit 74f87a6375c9fded81a6824c6d6317d08e6dd17d. The problematic commmit in Python 3.13.6 has been reverted, so we can now reenable the test suite. --- pkgs/development/python-modules/websockets/default.nix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/websockets/default.nix b/pkgs/development/python-modules/websockets/default.nix index 0a01b25a55c20..25ff8ada5b623 100644 --- a/pkgs/development/python-modules/websockets/default.nix +++ b/pkgs/development/python-modules/websockets/default.nix @@ -61,8 +61,7 @@ buildPythonPackage rec { ''; # Tests fail on Darwin with `OSError: AF_UNIX path too long` - # https://github.com/python-websockets/websockets/issues/1648 - doCheck = pythonOlder "3.13" && !stdenv.hostPlatform.isDarwin; + doCheck = !stdenv.hostPlatform.isDarwin; pythonImportsCheck = [ "websockets" ]; From fbd9a88c36b4f2051eb2fe5af5869acd21ea50a2 Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Sun, 10 Aug 2025 02:07:13 +0200 Subject: [PATCH 3/3] python3Packages.sentence-stream: relax regex constraint --- pkgs/development/python-modules/sentence-stream/default.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkgs/development/python-modules/sentence-stream/default.nix b/pkgs/development/python-modules/sentence-stream/default.nix index 3d7d2f8edba3b..0c6907dba076e 100644 --- a/pkgs/development/python-modules/sentence-stream/default.nix +++ b/pkgs/development/python-modules/sentence-stream/default.nix @@ -28,6 +28,10 @@ buildPythonPackage rec { regex ]; + pythonRelaxDeps = [ + "regex" + ]; + nativeCheckInputs = [ pytest-asyncio pytestCheckHook