Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Pyhon 3.13 CI #587

Open
wants to merge 28 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
207 changes: 147 additions & 60 deletions .github/workflows/ci.yml

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions continuous-delivery/build-wheels-manylinux2014-aarch64.sh
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp311*.whl

# Don't need to build wheels for Python 3.12 and later.
# The 3.11 wheel uses the stable ABI, so it works with newer versions too.
# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions.
/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel
auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp313*.whl

rm dist/*.whl
cp -rv wheelhouse/* dist/
Expand Down
3 changes: 3 additions & 0 deletions continuous-delivery/build-wheels-manylinux2014-x86_64.sh
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp311*.whl

# Don't need to build wheels for Python 3.12 and later.
# The 3.11 wheel uses the stable ABI, so it works with newer versions too.
# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions.
/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel
auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp313*.whl

rm dist/*.whl
cp -rv wheelhouse/* dist/
Expand Down
4 changes: 4 additions & 0 deletions continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@ auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp311*.whl
# Don't need to build wheels for Python 3.12 and later.
# The 3.11 wheel uses the stable ABI, so it works with newer versions too.

# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions.
/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel
auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp313*.whl

rm dist/*.whl
cp -rv wheelhouse/* dist/

Expand Down
4 changes: 4 additions & 0 deletions continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@ auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp311*.whl
# Don't need to build wheels for Python 3.12 and later.
# The 3.11 wheel uses the stable ABI, so it works with newer versions too.

# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions.
/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel
auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp313*.whl

rm dist/*.whl
cp -rv wheelhouse/* dist/

Expand Down
3 changes: 3 additions & 0 deletions continuous-delivery/build-wheels-osx.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,7 @@ set -ex
# Don't need to build wheels for Python 3.12 and later.
# The 3.11 wheel uses the stable ABI, so it works with newer versions too.

# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions.
/Library/Frameworks/Python.framework/Versions/3.13/bin/python3 setup.py sdist bdist_wheel

#now you just need to run twine (that's in a different script)
6 changes: 6 additions & 0 deletions continuous-delivery/build-wheels-win32.bat
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@
"C:\Program Files (x86)\Python310-32\python.exe" setup.py sdist bdist_wheel || goto error
"C:\Program Files (x86)\Python311-32\python.exe" setup.py sdist bdist_wheel || goto error

:: Don't need to build wheels for Python 3.12 and later.
:: The 3.11 wheel uses the stable ABI, so it works with newer versions too.

:: We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions.
"C:\Program Files (x86)\Python313-32\python.exe" setup.py sdist bdist_wheel || goto error

goto :EOF

:error
Expand Down
6 changes: 6 additions & 0 deletions continuous-delivery/build-wheels-win64.bat
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@
"C:\Program Files\Python310\python.exe" setup.py sdist bdist_wheel || goto error
"C:\Program Files\Python311\python.exe" setup.py sdist bdist_wheel || goto error

:: Don't need to build wheels for Python 3.12 and later.
:: The 3.11 wheel uses the stable ABI, so it works with newer versions too.

:: We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions.
"C:\Program Files\Python313\python.exe" setup.py sdist bdist_wheel || goto error

goto :EOF

:error
Expand Down
2 changes: 1 addition & 1 deletion crt/aws-c-mqtt
Submodule aws-c-mqtt updated 45 files
+12 −3 .github/workflows/ci.yml
+4 −6 .github/workflows/clang-format.yml
+2 −0 CMakeLists.txt
+29 −0 bin/elastishadow/CMakeLists.txt
+1,272 −0 bin/elastishadow/main.c
+47 −0 format-check.py
+0 −24 format-check.sh
+9 −0 include/aws/mqtt/mqtt.h
+54 −0 include/aws/mqtt/private/client_impl.h
+24 −0 include/aws/mqtt/private/client_impl_shared.h
+204 −0 include/aws/mqtt/private/mqtt311_listener.h
+220 −0 include/aws/mqtt/private/request-response/protocol_adapter.h
+23 −0 include/aws/mqtt/private/request-response/request_response_client.h
+264 −0 include/aws/mqtt/private/request-response/subscription_manager.h
+2 −0 include/aws/mqtt/private/shared.h
+274 −0 include/aws/mqtt/request-response/request_response_client.h
+3 −0 include/aws/mqtt/v5/mqtt5_client.h
+176 −58 source/client.c
+6 −0 source/client_channel_handler.c
+9 −0 source/client_impl_shared.c
+25 −0 source/mqtt.c
+329 −0 source/mqtt311_listener.c
+2 −3 source/packets.c
+964 −0 source/request-response/protocol_adapter.c
+2,276 −0 source/request-response/request_response_client.c
+822 −0 source/request-response/subscription_manager.c
+1 −1 source/shared.c
+1 −1 source/v5/mqtt5_client.c
+1 −0 source/v5/mqtt5_listener.c
+14 −0 source/v5/mqtt5_to_mqtt3_adapter.c
+135 −3 tests/CMakeLists.txt
+1,784 −0 tests/request-response/protocol_adapter_tests.c
+3,151 −0 tests/request-response/request_response_client_tests.c
+2,877 −0 tests/request-response/subscription_manager_tests.c
+739 −998 tests/v3/connection_state_test.c
+488 −0 tests/v3/mqtt311_listener_test.c
+582 −0 tests/v3/mqtt311_testing_utils.c
+155 −0 tests/v3/mqtt311_testing_utils.h
+51 −2 tests/v3/mqtt_mock_server_handler.c
+18 −0 tests/v3/mqtt_mock_server_handler.h
+22 −42 tests/v5/mqtt5_client_tests.c
+26 −0 tests/v5/mqtt5_testing_utils.c
+20 −0 tests/v5/mqtt5_testing_utils.h
+0 −4 tests/v5/mqtt5_to_mqtt3_adapter_tests.c
+2 −1 tests/v5/mqtt5_topic_alias_tests.c
2 changes: 1 addition & 1 deletion crt/aws-lc
2 changes: 1 addition & 1 deletion crt/s2n
Submodule s2n updated from 87f4a0 to 08d413
5 changes: 4 additions & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -403,7 +403,10 @@ def awscrt_ext():
else:
extra_link_args += ['-Wl,--fatal-warnings']

if sys.version_info >= (3, 11):
if sys.version_info >= (3, 13):
define_macros.append(('Py_LIMITED_API', '0x030D0000'))
py_limited_api = True
elif sys.version_info >= (3, 11):
define_macros.append(('Py_LIMITED_API', '0x030B0000'))
py_limited_api = True

Expand Down
14 changes: 13 additions & 1 deletion source/http_stream.c
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,19 @@ static void s_on_stream_complete(struct aws_http_stream *native_stream, int erro
}

/* DECREF python self, we don't need to force it to stay alive any longer. */
Py_DECREF(PyWeakref_GetObject(stream->self_proxy));
PyObject *self = Py_None;
#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */
int result_getref = PyWeakref_GetRef(stream->self_proxy, &self); /* strong reference */
/* Ignore error, stream is already complete */
(void)result_getref;
#else
/* PyWeakref_GetObject is deprecated since python 3.13 */
self = PyWeakref_GetObject(stream->self_proxy); /* borrowed reference */
#endif
Py_XDECREF(self);
#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */
Py_XDECREF(self);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you end up decreffing it twice in 3.13 branch?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, one is the original decref that we need to do, and the second decref is for the strong reference that the new API returned.

#endif

PyGILState_Release(state);
/*************** GIL RELEASE ***************/
Expand Down
102 changes: 94 additions & 8 deletions source/mqtt_client_connection.c
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,17 @@ static void s_on_connection_success(
return; /* Python has shut down. Nothing matters anymore, but don't crash */
}

PyObject *self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */
PyObject *self = Py_None;
#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */
if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */
PyErr_WriteUnraisable(PyErr_Occurred());
goto on_done;
}
#else
/* PyWeakref_GetObject is deprecated since python 3.13 */
self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */
#endif

if (self != Py_None) {
PyObject *success_result =
PyObject_CallMethod(self, "_on_connection_success", "(iN)", return_code, PyBool_FromLong(session_present));
Expand All @@ -150,7 +160,11 @@ static void s_on_connection_success(
PyErr_WriteUnraisable(PyErr_Occurred());
}
}

goto on_done; /* fixes unused label waring */
on_done:
#if PY_VERSION_HEX >= 0x030D0000
Py_XDECREF(self);
#endif
PyGILState_Release(state);
}

Expand All @@ -167,7 +181,17 @@ static void s_on_connection_failure(struct aws_mqtt_client_connection *connectio
return; /* Python has shut down. Nothing matters anymore, but don't crash */
}

PyObject *self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */
PyObject *self = Py_None;
#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */
if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */
PyErr_WriteUnraisable(PyErr_Occurred());
goto on_done;
}
#else
/* PyWeakref_GetObject is deprecated since python 3.13 */
self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */
#endif

if (self != Py_None) {
PyObject *success_result = PyObject_CallMethod(self, "_on_connection_failure", "(i)", error_code);
if (success_result) {
Expand All @@ -177,6 +201,11 @@ static void s_on_connection_failure(struct aws_mqtt_client_connection *connectio
}
}

goto on_done; /* fixes unused label waring */
on_done:
#if PY_VERSION_HEX >= 0x030D0000
Py_XDECREF(self);
#endif
PyGILState_Release(state);
}

Expand All @@ -194,7 +223,17 @@ static void s_on_connection_interrupted(struct aws_mqtt_client_connection *conne
}

/* Ensure that python class is still alive */
PyObject *self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */
PyObject *self = Py_None;
#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */
if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */
PyErr_WriteUnraisable(PyErr_Occurred());
goto on_done;
}
#else
/* PyWeakref_GetObject is deprecated since python 3.13 */
self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */
#endif

if (self != Py_None) {
PyObject *result = PyObject_CallMethod(self, "_on_connection_interrupted", "(i)", error_code);
if (result) {
Expand All @@ -204,6 +243,11 @@ static void s_on_connection_interrupted(struct aws_mqtt_client_connection *conne
}
}

goto on_done; /* fixes unused label waring */
on_done:
#if PY_VERSION_HEX >= 0x030D0000
Py_XDECREF(self);
#endif
PyGILState_Release(state);
}

Expand All @@ -227,7 +271,17 @@ static void s_on_connection_resumed(
}

/* Ensure that python class is still alive */
PyObject *self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */
PyObject *self = Py_None;
#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */
if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */
PyErr_WriteUnraisable(PyErr_Occurred());
goto on_done;
}
#else
/* PyWeakref_GetObject is deprecated since python 3.13 */
self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */
#endif

if (self != Py_None) {
PyObject *result =
PyObject_CallMethod(self, "_on_connection_resumed", "(iN)", return_code, PyBool_FromLong(session_present));
Expand All @@ -237,7 +291,11 @@ static void s_on_connection_resumed(
PyErr_WriteUnraisable(PyErr_Occurred());
}
}

goto on_done; /* fixes unused label waring */
on_done:
#if PY_VERSION_HEX >= 0x030D0000
Py_XDECREF(self);
#endif
PyGILState_Release(state);
}

Expand All @@ -258,7 +316,17 @@ static void s_on_connection_closed(

struct mqtt_connection_binding *py_connection = userdata;
/* Ensure that python class is still alive */
PyObject *self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */
PyObject *self = Py_None;
#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */
if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */
PyErr_WriteUnraisable(PyErr_Occurred());
goto on_done;
}
#else
/* PyWeakref_GetObject is deprecated since python 3.13 */
self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */
#endif

if (self != Py_None) {
PyObject *result = PyObject_CallMethod(self, "_on_connection_closed", "()");
if (result) {
Expand All @@ -268,6 +336,11 @@ static void s_on_connection_closed(
}
}

goto on_done; /* fixes unused label waring */
on_done:
#if PY_VERSION_HEX >= 0x030D0000
Py_XDECREF(self);
#endif
PyGILState_Release(state);
}

Expand Down Expand Up @@ -535,7 +608,17 @@ static void s_ws_handshake_transform(
}

/* Ensure python mqtt connection object is still alive */
PyObject *connection_py = PyWeakref_GetObject(connection_binding->self_proxy); /* borrowed reference */
PyObject *connection_py = Py_None;
#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */
if (PyWeakref_GetRef(connection_binding->self_proxy, &connection_py) < 0) { /* strong reference */
aws_raise_error(AWS_ERROR_INVALID_STATE);
goto done;
}
#else
/* PyWeakref_GetObject is deprecated since python 3.13 */
connection_py = PyWeakref_GetObject(connection_binding->self_proxy); /* borrowed reference */
#endif

if (connection_py == Py_None) {
aws_raise_error(AWS_ERROR_INVALID_STATE);
goto done;
Expand Down Expand Up @@ -593,6 +676,9 @@ static void s_ws_handshake_transform(
done:;
/* Save off error code, so it doesn't got stomped before we pass it to callback*/
int error_code = aws_last_error();
#if PY_VERSION_HEX >= 0x030D0000
Py_XDECREF(connection_py);
#endif

if (ws_transform_capsule) {
Py_DECREF(ws_transform_capsule);
Expand Down
45 changes: 10 additions & 35 deletions test/test_mqtt5_credentials.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,40 +197,6 @@ def test_mqtt5_ws_cred_static(self):
input_role_secret_access_key,
input_role_session_token
)
credentials = auth.AwsCredentialsProvider.new_default_chain()

def sign_function(transform_args, **kwargs):
signing_config = auth.AwsSigningConfig(
algorithm=auth.AwsSigningAlgorithm.V4,
signature_type=auth.AwsSignatureType.HTTP_REQUEST_QUERY_PARAMS,
credentials_provider=credentials,
region=input_region,
service="iotdevicegateway",
omit_session_token=True
)
signing_future = auth.aws_sign_request(
http_request=transform_args.http_request,
signing_config=signing_config)
signing_future.add_done_callback(lambda x: transform_args.set_done(x.exception()))
client_options.websocket_handshake_transform = sign_function
client_options.tls_ctx = io.ClientTlsContext(io.TlsContextOptions())

callbacks = Mqtt5TestCallbacks()
client = self._create_client(client_options=client_options, callbacks=callbacks)
client.start()
callbacks.future_connection_success.result(TIMEOUT)
client.stop()
callbacks.future_stopped.result(TIMEOUT)

def test_mqtt5_ws_cred_default(self):
input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST")
input_region = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_REGION")

client_options = mqtt5.ClientOptions(
host_name=input_host_name,
port=443
)
credentials = auth.AwsCredentialsProvider.new_default_chain()

def sign_function(transform_args, **kwargs):
signing_config = auth.AwsSigningConfig(
Expand Down Expand Up @@ -380,6 +346,12 @@ def sign_function(transform_args, **kwargs):
callbacks.future_stopped.result(TIMEOUT)

def test_mqtt5_ws_cred_environment(self):
self._test_mqtt5_ws_cred_environment(use_default_chain=False)

def test_mqtt5_ws_cred_default_chain(self):
self._test_mqtt5_ws_cred_environment(use_default_chain=True)

def _test_mqtt5_ws_cred_environment(self, use_default_chain):
input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST")
input_access_key = _get_env_variable("AWS_TEST_MQTT5_ROLE_CREDENTIAL_ACCESS_KEY")
input_secret_access_key = _get_env_variable("AWS_TEST_MQTT5_ROLE_CREDENTIAL_SECRET_ACCESS_KEY")
Expand All @@ -399,7 +371,10 @@ def test_mqtt5_ws_cred_environment(self):
os.environ["AWS_SECRET_ACCESS_KEY"] = input_secret_access_key
os.environ["AWS_SESSION_TOKEN"] = input_session_token
# This should load the environment variables we just set
credentials = auth.AwsCredentialsProvider.new_environment()
if use_default_chain:
credentials = auth.AwsCredentialsProvider.new_default_chain()
else:
credentials = auth.AwsCredentialsProvider.new_environment()

def sign_function(transform_args, **kwargs):
signing_config = auth.AwsSigningConfig(
Expand Down
Loading
Loading