From 70de9dfbc7fef9d5308df39b9b3e24c7c2540afa Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 22 Aug 2024 15:17:30 -0700 Subject: [PATCH 01/27] python 313 CI --- .github/workflows/ci.yml | 4 +++- crt/aws-c-compression | 2 +- crt/aws-lc | 2 +- crt/s2n | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6786da5f..be20926e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ on: - 'docs' env: - BUILDER_VERSION: v0.9.62 + BUILDER_VERSION: v0.9.63 BUILDER_SOURCE: releases BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net PACKAGE_NAME: aws-crt-python @@ -53,6 +53,7 @@ jobs: - cp310-cp310 - cp311-cp311 - cp312-cp312 + - cp313-cp313 steps: # Only aarch64 needs this, but it doesn't hurt anything - name: Install qemu/docker @@ -78,6 +79,7 @@ jobs: - cp310-cp310 - cp311-cp311 - cp312-cp312 + - cp313-cp313 steps: # Only aarch64 needs this, but it doesn't hurt anything - name: Install qemu/docker diff --git a/crt/aws-c-compression b/crt/aws-c-compression index ea1d421a..f36d0167 160000 --- a/crt/aws-c-compression +++ b/crt/aws-c-compression @@ -1 +1 @@ -Subproject commit ea1d421a421ad83a540309a94c38d50b6a5d836b +Subproject commit f36d01672d61e49d96a777870d456f66fa391cd4 diff --git a/crt/aws-lc b/crt/aws-lc index 05747780..2f187975 160000 --- a/crt/aws-lc +++ b/crt/aws-lc @@ -1 +1 @@ -Subproject commit 05747780676652f41d0b9c570a495e4bb6608560 +Subproject commit 2f1879759b2e0fc70592665bdf10087b64f44b7d diff --git a/crt/s2n b/crt/s2n index 79c0f1b4..87f4a058 160000 --- a/crt/s2n +++ b/crt/s2n @@ -1 +1 @@ -Subproject commit 79c0f1b434742d9f1152c48d3781433649f6f8fe +Subproject commit 87f4a0585dc3056433f193b9305f587cff239be3 From d2b0b271d74eddb885b36d9358d59133bc848cea Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 26 Aug 2024 14:13:10 -0700 Subject: [PATCH 02/27] wip test --- source/http_stream.c | 2 +- source/mqtt_client_connection.c | 27 ++++++++++++++++++++------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/source/http_stream.c b/source/http_stream.c index bf702186..09943d89 100644 --- a/source/http_stream.c +++ b/source/http_stream.c @@ -203,7 +203,7 @@ static void s_on_stream_complete(struct aws_http_stream *native_stream, int erro } /* DECREF python self, we don't need to force it to stay alive any longer. */ - Py_DECREF(PyWeakref_GetObject(stream->self_proxy)); + Py_XDECREF(stream->self_proxy); PyGILState_Release(state); /*************** GIL RELEASE ***************/ diff --git a/source/mqtt_client_connection.c b/source/mqtt_client_connection.c index 9eb73a95..acaae556 100644 --- a/source/mqtt_client_connection.c +++ b/source/mqtt_client_connection.c @@ -140,7 +140,17 @@ static void s_on_connection_success( return; /* Python has shut down. Nothing matters anymore, but don't crash */ } - PyObject *self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + PyObject *self = Py_None; +#if PY_VERSION_HEX >= 0x030D0000 // Check if Python version is 3.13 or higher + if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */ + PyErr_WriteUnraisable(PyErr_Occurred()); + goto on_done; + } +#else + /* PyWeakref_GetObject is deprecated since python 3.13 */ + PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ +#endif + if (self != Py_None) { PyObject *success_result = PyObject_CallMethod(self, "_on_connection_success", "(iN)", return_code, PyBool_FromLong(session_present)); @@ -150,7 +160,10 @@ static void s_on_connection_success( PyErr_WriteUnraisable(PyErr_Occurred()); } } - +#if PY_VERSION_HEX >= 0x030D0000 + Py_DECREF(self); +#endif +on_done: PyGILState_Release(state); } @@ -167,7 +180,7 @@ static void s_on_connection_failure(struct aws_mqtt_client_connection *connectio return; /* Python has shut down. Nothing matters anymore, but don't crash */ } - PyObject *self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + PyObject *self = Py_None; // PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ if (self != Py_None) { PyObject *success_result = PyObject_CallMethod(self, "_on_connection_failure", "(i)", error_code); if (success_result) { @@ -194,7 +207,7 @@ static void s_on_connection_interrupted(struct aws_mqtt_client_connection *conne } /* Ensure that python class is still alive */ - PyObject *self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + PyObject *self = Py_None; // PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ if (self != Py_None) { PyObject *result = PyObject_CallMethod(self, "_on_connection_interrupted", "(i)", error_code); if (result) { @@ -227,7 +240,7 @@ static void s_on_connection_resumed( } /* Ensure that python class is still alive */ - PyObject *self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + PyObject *self = Py_None; // PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ if (self != Py_None) { PyObject *result = PyObject_CallMethod(self, "_on_connection_resumed", "(iN)", return_code, PyBool_FromLong(session_present)); @@ -258,7 +271,7 @@ static void s_on_connection_closed( struct mqtt_connection_binding *py_connection = userdata; /* Ensure that python class is still alive */ - PyObject *self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + PyObject *self = Py_None; // PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ if (self != Py_None) { PyObject *result = PyObject_CallMethod(self, "_on_connection_closed", "()"); if (result) { @@ -535,7 +548,7 @@ static void s_ws_handshake_transform( } /* Ensure python mqtt connection object is still alive */ - PyObject *connection_py = PyWeakref_GetObject(connection_binding->self_proxy); /* borrowed reference */ + PyObject *connection_py = Py_None; // PyWeakref_GetObject(connection_binding->self_proxy); /* borrowed reference */ if (connection_py == Py_None) { aws_raise_error(AWS_ERROR_INVALID_STATE); goto done; From f4f6ea242929b799d90ebf63cb4192139d47d090 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 26 Aug 2024 15:45:56 -0700 Subject: [PATCH 03/27] fix warnings --- setup.py | 5 +- source/mqtt_client_connection.c | 86 ++++++++++++++++++++++++++++++--- 2 files changed, 82 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 32873b2b..9a7b674f 100644 --- a/setup.py +++ b/setup.py @@ -396,7 +396,10 @@ def awscrt_ext(): else: extra_link_args += ['-Wl,--fatal-warnings'] - if sys.version_info >= (3, 11): + if sys.version_info >= (3,13): + define_macros.append(('Py_LIMITED_API', '0x030D0000')) + py_limited_api = True + elif sys.version_info >= (3,11): define_macros.append(('Py_LIMITED_API', '0x030B0000')) py_limited_api = True diff --git a/source/mqtt_client_connection.c b/source/mqtt_client_connection.c index acaae556..df1eaed6 100644 --- a/source/mqtt_client_connection.c +++ b/source/mqtt_client_connection.c @@ -160,10 +160,10 @@ static void s_on_connection_success( PyErr_WriteUnraisable(PyErr_Occurred()); } } +on_done: #if PY_VERSION_HEX >= 0x030D0000 - Py_DECREF(self); + Py_XDECREF(self); #endif -on_done: PyGILState_Release(state); } @@ -180,7 +180,17 @@ static void s_on_connection_failure(struct aws_mqtt_client_connection *connectio return; /* Python has shut down. Nothing matters anymore, but don't crash */ } - PyObject *self = Py_None; // PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + PyObject *self = Py_None; +#if PY_VERSION_HEX >= 0x030D0000 // Check if Python version is 3.13 or higher + if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */ + PyErr_WriteUnraisable(PyErr_Occurred()); + goto on_done; + } +#else + /* PyWeakref_GetObject is deprecated since python 3.13 */ + PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ +#endif + if (self != Py_None) { PyObject *success_result = PyObject_CallMethod(self, "_on_connection_failure", "(i)", error_code); if (success_result) { @@ -190,6 +200,10 @@ static void s_on_connection_failure(struct aws_mqtt_client_connection *connectio } } +on_done: +#if PY_VERSION_HEX >= 0x030D0000 + Py_XDECREF(self); +#endif PyGILState_Release(state); } @@ -207,7 +221,17 @@ static void s_on_connection_interrupted(struct aws_mqtt_client_connection *conne } /* Ensure that python class is still alive */ - PyObject *self = Py_None; // PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + PyObject *self = Py_None; +#if PY_VERSION_HEX >= 0x030D0000 // Check if Python version is 3.13 or higher + if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */ + PyErr_WriteUnraisable(PyErr_Occurred()); + goto on_done; + } +#else + /* PyWeakref_GetObject is deprecated since python 3.13 */ + PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ +#endif + if (self != Py_None) { PyObject *result = PyObject_CallMethod(self, "_on_connection_interrupted", "(i)", error_code); if (result) { @@ -217,6 +241,10 @@ static void s_on_connection_interrupted(struct aws_mqtt_client_connection *conne } } +on_done: +#if PY_VERSION_HEX >= 0x030D0000 + Py_XDECREF(self); +#endif PyGILState_Release(state); } @@ -240,7 +268,17 @@ static void s_on_connection_resumed( } /* Ensure that python class is still alive */ - PyObject *self = Py_None; // PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + PyObject *self = Py_None; +#if PY_VERSION_HEX >= 0x030D0000 // Check if Python version is 3.13 or higher + if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */ + PyErr_WriteUnraisable(PyErr_Occurred()); + goto on_done; + } +#else + /* PyWeakref_GetObject is deprecated since python 3.13 */ + PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ +#endif + if (self != Py_None) { PyObject *result = PyObject_CallMethod(self, "_on_connection_resumed", "(iN)", return_code, PyBool_FromLong(session_present)); @@ -250,7 +288,10 @@ static void s_on_connection_resumed( PyErr_WriteUnraisable(PyErr_Occurred()); } } - +on_done: +#if PY_VERSION_HEX >= 0x030D0000 + Py_XDECREF(self); +#endif PyGILState_Release(state); } @@ -271,7 +312,17 @@ static void s_on_connection_closed( struct mqtt_connection_binding *py_connection = userdata; /* Ensure that python class is still alive */ - PyObject *self = Py_None; // PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + PyObject *self = Py_None; +#if PY_VERSION_HEX >= 0x030D0000 // Check if Python version is 3.13 or higher + if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */ + PyErr_WriteUnraisable(PyErr_Occurred()); + goto on_done; + } +#else + /* PyWeakref_GetObject is deprecated since python 3.13 */ + PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ +#endif + if (self != Py_None) { PyObject *result = PyObject_CallMethod(self, "_on_connection_closed", "()"); if (result) { @@ -281,6 +332,11 @@ static void s_on_connection_closed( } } + +on_done: +#if PY_VERSION_HEX >= 0x030D0000 + Py_XDECREF(self); +#endif PyGILState_Release(state); } @@ -548,7 +604,17 @@ static void s_ws_handshake_transform( } /* Ensure python mqtt connection object is still alive */ - PyObject *connection_py = Py_None; // PyWeakref_GetObject(connection_binding->self_proxy); /* borrowed reference */ + PyObject *connection_py = Py_None; +#if PY_VERSION_HEX >= 0x030D0000 // Check if Python version is 3.13 or higher + if (PyWeakref_GetRef(connection_binding->self_proxy, &connection_py) < 0) { /* strong reference */ + aws_raise_error(AWS_ERROR_INVALID_STATE); + goto done; + } +#else + /* PyWeakref_GetObject is deprecated since python 3.13 */ + PyWeakref_GetObject(connection_binding->self_proxy); /* borrowed reference */ +#endif + if (connection_py == Py_None) { aws_raise_error(AWS_ERROR_INVALID_STATE); goto done; @@ -606,6 +672,10 @@ static void s_ws_handshake_transform( done:; /* Save off error code, so it doesn't got stomped before we pass it to callback*/ int error_code = aws_last_error(); +#if PY_VERSION_HEX >= 0x030D0000 + Py_XDECREF(connection_py); +#endif + if (ws_transform_capsule) { Py_DECREF(ws_transform_capsule); From d19acc871f813490faadc61e1021afe8364c78d5 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 09:19:06 -0700 Subject: [PATCH 04/27] new changes --- crt/aws-c-auth | 2 +- crt/aws-c-cal | 2 +- crt/aws-c-common | 2 +- crt/aws-c-http | 2 +- crt/aws-c-mqtt | 2 +- crt/aws-c-s3 | 2 +- crt/aws-checksums | 2 +- crt/aws-lc | 2 +- crt/s2n | 2 +- source/http_stream.c | 13 ++++++++++++- source/mqtt_client_connection.c | 22 ++++++++++------------ 11 files changed, 31 insertions(+), 22 deletions(-) diff --git a/crt/aws-c-auth b/crt/aws-c-auth index 52bf5916..48d647bf 160000 --- a/crt/aws-c-auth +++ b/crt/aws-c-auth @@ -1 +1 @@ -Subproject commit 52bf591613d1a001c43ec99af7376f871759c5fe +Subproject commit 48d647bf43f8872e4dc5ec6343b0c5974195fbdd diff --git a/crt/aws-c-cal b/crt/aws-c-cal index 77ca3aea..2cb1d2ea 160000 --- a/crt/aws-c-cal +++ b/crt/aws-c-cal @@ -1 +1 @@ -Subproject commit 77ca3aea879bc768082fe7ec715adcde8e98c332 +Subproject commit 2cb1d2eac925e2dbc45025eb89af82bd790c23a0 diff --git a/crt/aws-c-common b/crt/aws-c-common index 672cc003..b9959f59 160000 --- a/crt/aws-c-common +++ b/crt/aws-c-common @@ -1 +1 @@ -Subproject commit 672cc0032eb28d69fbdd22c9463253c89d7a6f30 +Subproject commit b9959f5922a4b969beab8f0b99aa0b34bc9ee55c diff --git a/crt/aws-c-http b/crt/aws-c-http index 4e74ab1e..6068653e 160000 --- a/crt/aws-c-http +++ b/crt/aws-c-http @@ -1 +1 @@ -Subproject commit 4e74ab1e3702763e0b87bd1752f5a37c2f0400ac +Subproject commit 6068653e1d582bd8e7d1c9f81f86beaf10444e3d diff --git a/crt/aws-c-mqtt b/crt/aws-c-mqtt index ed7bbd68..c43232c1 160000 --- a/crt/aws-c-mqtt +++ b/crt/aws-c-mqtt @@ -1 +1 @@ -Subproject commit ed7bbd68c03d7022c915a2924740ab7992ad2311 +Subproject commit c43232c1bc378344bb7245d7fcb167410f3fe931 diff --git a/crt/aws-c-s3 b/crt/aws-c-s3 index 0ab4d58e..aede1d8c 160000 --- a/crt/aws-c-s3 +++ b/crt/aws-c-s3 @@ -1 +1 @@ -Subproject commit 0ab4d58ef0bd97970d43828cb6b57a3de5747343 +Subproject commit aede1d8c24f9f580d5a96c089878e9b258b88d04 diff --git a/crt/aws-checksums b/crt/aws-checksums index aac442a2..ce04ab00 160000 --- a/crt/aws-checksums +++ b/crt/aws-checksums @@ -1 +1 @@ -Subproject commit aac442a2dbbb5e72d0a3eca8313cf65e7e1cac2f +Subproject commit ce04ab00b3ecc41912f478bfedca39f8e1919d6b diff --git a/crt/aws-lc b/crt/aws-lc index 2f187975..d3a598c1 160000 --- a/crt/aws-lc +++ b/crt/aws-lc @@ -1 +1 @@ -Subproject commit 2f1879759b2e0fc70592665bdf10087b64f44b7d +Subproject commit d3a598c1b419d49b5b08f0677add4581572e2edc diff --git a/crt/s2n b/crt/s2n index 87f4a058..08d413a0 160000 --- a/crt/s2n +++ b/crt/s2n @@ -1 +1 @@ -Subproject commit 87f4a0585dc3056433f193b9305f587cff239be3 +Subproject commit 08d413a0b9b3226e775a38f04e3cf02230cc97c4 diff --git a/source/http_stream.c b/source/http_stream.c index 09943d89..f6f09bde 100644 --- a/source/http_stream.c +++ b/source/http_stream.c @@ -203,7 +203,18 @@ static void s_on_stream_complete(struct aws_http_stream *native_stream, int erro } /* DECREF python self, we don't need to force it to stay alive any longer. */ - Py_XDECREF(stream->self_proxy); + PyObject *self = Py_None; +#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ + /* Ignore error, stream is already complete */ + PyWeakref_GetRef(stream->self_proxy, &self) < 0);/* strong reference */ +#else + /* PyWeakref_GetObject is deprecated since python 3.13 */ + PyWeakref_GetObject(stream->self_proxy); /* borrowed reference */ +#endif + Py_XDECREF(self); +#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ + Py_XDECREF(self); +#endif PyGILState_Release(state); /*************** GIL RELEASE ***************/ diff --git a/source/mqtt_client_connection.c b/source/mqtt_client_connection.c index df1eaed6..ab5c5291 100644 --- a/source/mqtt_client_connection.c +++ b/source/mqtt_client_connection.c @@ -141,7 +141,7 @@ static void s_on_connection_success( } PyObject *self = Py_None; -#if PY_VERSION_HEX >= 0x030D0000 // Check if Python version is 3.13 or higher +#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */ PyErr_WriteUnraisable(PyErr_Occurred()); goto on_done; @@ -180,8 +180,8 @@ static void s_on_connection_failure(struct aws_mqtt_client_connection *connectio return; /* Python has shut down. Nothing matters anymore, but don't crash */ } - PyObject *self = Py_None; -#if PY_VERSION_HEX >= 0x030D0000 // Check if Python version is 3.13 or higher + PyObject *self = Py_None; +#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */ PyErr_WriteUnraisable(PyErr_Occurred()); goto on_done; @@ -222,7 +222,7 @@ static void s_on_connection_interrupted(struct aws_mqtt_client_connection *conne /* Ensure that python class is still alive */ PyObject *self = Py_None; -#if PY_VERSION_HEX >= 0x030D0000 // Check if Python version is 3.13 or higher +#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */ PyErr_WriteUnraisable(PyErr_Occurred()); goto on_done; @@ -268,8 +268,8 @@ static void s_on_connection_resumed( } /* Ensure that python class is still alive */ - PyObject *self = Py_None; -#if PY_VERSION_HEX >= 0x030D0000 // Check if Python version is 3.13 or higher + PyObject *self = Py_None; +#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */ PyErr_WriteUnraisable(PyErr_Occurred()); goto on_done; @@ -312,8 +312,8 @@ static void s_on_connection_closed( struct mqtt_connection_binding *py_connection = userdata; /* Ensure that python class is still alive */ - PyObject *self = Py_None; -#if PY_VERSION_HEX >= 0x030D0000 // Check if Python version is 3.13 or higher + PyObject *self = Py_None; +#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */ PyErr_WriteUnraisable(PyErr_Occurred()); goto on_done; @@ -332,7 +332,6 @@ static void s_on_connection_closed( } } - on_done: #if PY_VERSION_HEX >= 0x030D0000 Py_XDECREF(self); @@ -604,8 +603,8 @@ static void s_ws_handshake_transform( } /* Ensure python mqtt connection object is still alive */ - PyObject *connection_py = Py_None; -#if PY_VERSION_HEX >= 0x030D0000 // Check if Python version is 3.13 or higher + PyObject *connection_py = Py_None; +#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ if (PyWeakref_GetRef(connection_binding->self_proxy, &connection_py) < 0) { /* strong reference */ aws_raise_error(AWS_ERROR_INVALID_STATE); goto done; @@ -676,7 +675,6 @@ done:; Py_XDECREF(connection_py); #endif - if (ws_transform_capsule) { Py_DECREF(ws_transform_capsule); } else if (ws_transform_data) { From d7d4bf980d03073e2edf1397a32639bdf0735440 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 09:26:39 -0700 Subject: [PATCH 05/27] builder version --- .github/workflows/ci.yml | 2 +- continuous-delivery/build-wheels-manylinux2014-aarch64.sh | 3 +++ continuous-delivery/build-wheels-manylinux2014-x86_64.sh | 3 +++ continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh | 4 ++++ continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh | 4 ++++ continuous-delivery/build-wheels-osx.sh | 3 +++ continuous-delivery/build-wheels-win32.bat | 6 ++++++ continuous-delivery/build-wheels-win64.bat | 6 ++++++ 8 files changed, 30 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be20926e..324561dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ on: - 'docs' env: - BUILDER_VERSION: v0.9.63 + BUILDER_VERSION: v0.9.66 BUILDER_SOURCE: releases BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net PACKAGE_NAME: aws-crt-python diff --git a/continuous-delivery/build-wheels-manylinux2014-aarch64.sh b/continuous-delivery/build-wheels-manylinux2014-aarch64.sh index a140b7d6..8ff18c89 100755 --- a/continuous-delivery/build-wheels-manylinux2014-aarch64.sh +++ b/continuous-delivery/build-wheels-manylinux2014-aarch64.sh @@ -21,6 +21,9 @@ auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp311*.whl # Don't need to build wheels for Python 3.12 and later. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. +# We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp313*.whl rm dist/*.whl cp -rv wheelhouse/* dist/ diff --git a/continuous-delivery/build-wheels-manylinux2014-x86_64.sh b/continuous-delivery/build-wheels-manylinux2014-x86_64.sh index 41ce9fde..2714fe9e 100755 --- a/continuous-delivery/build-wheels-manylinux2014-x86_64.sh +++ b/continuous-delivery/build-wheels-manylinux2014-x86_64.sh @@ -21,6 +21,9 @@ auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp311*.whl # Don't need to build wheels for Python 3.12 and later. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. +# We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp313*.whl rm dist/*.whl cp -rv wheelhouse/* dist/ diff --git a/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh b/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh index 0cbc1a19..c2f13cff 100755 --- a/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh +++ b/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh @@ -22,6 +22,10 @@ auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp311*.whl # Don't need to build wheels for Python 3.12 and later. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. +# We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp313*.whl + rm dist/*.whl cp -rv wheelhouse/* dist/ diff --git a/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh b/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh index 6637d95a..250cc5ba 100755 --- a/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh +++ b/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh @@ -22,6 +22,10 @@ auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp311*.whl # Don't need to build wheels for Python 3.12 and later. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. +# We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp313*.whl + rm dist/*.whl cp -rv wheelhouse/* dist/ diff --git a/continuous-delivery/build-wheels-osx.sh b/continuous-delivery/build-wheels-osx.sh index a3a92bc6..1faf2688 100755 --- a/continuous-delivery/build-wheels-osx.sh +++ b/continuous-delivery/build-wheels-osx.sh @@ -14,4 +14,7 @@ set -ex # Don't need to build wheels for Python 3.12 and later. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. +# We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +/Library/Frameworks/Python.framework/Versions/3.13/bin/python3 setup.py sdist bdist_wheel + #now you just need to run twine (that's in a different script) diff --git a/continuous-delivery/build-wheels-win32.bat b/continuous-delivery/build-wheels-win32.bat index 6a3e5397..52b82e49 100644 --- a/continuous-delivery/build-wheels-win32.bat +++ b/continuous-delivery/build-wheels-win32.bat @@ -7,6 +7,12 @@ "C:\Program Files (x86)\Python310-32\python.exe" setup.py sdist bdist_wheel || goto error "C:\Program Files (x86)\Python311-32\python.exe" setup.py sdist bdist_wheel || goto error +:: Don't need to build wheels for Python 3.12 and later. +:: The 3.11 wheel uses the stable ABI, so it works with newer versions too. + +:: We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +"C:\Program Files (x86)\Python313-32\python.exe" setup.py sdist bdist_wheel || goto error + goto :EOF :error diff --git a/continuous-delivery/build-wheels-win64.bat b/continuous-delivery/build-wheels-win64.bat index 5a862e6e..492fe204 100644 --- a/continuous-delivery/build-wheels-win64.bat +++ b/continuous-delivery/build-wheels-win64.bat @@ -6,6 +6,12 @@ "C:\Program Files\Python310\python.exe" setup.py sdist bdist_wheel || goto error "C:\Program Files\Python311\python.exe" setup.py sdist bdist_wheel || goto error +:: Don't need to build wheels for Python 3.12 and later. +:: The 3.11 wheel uses the stable ABI, so it works with newer versions too. + +:: We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +"C:\Program Files\Python313\python.exe" setup.py sdist bdist_wheel || goto error + goto :EOF :error From d269cc96dd094e2bc9c4e1383092df915aef9a9e Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 09:44:08 -0700 Subject: [PATCH 06/27] try new creds --- .github/workflows/ci.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 324561dc..5906be60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,9 +13,8 @@ env: PACKAGE_NAME: aws-crt-python LINUX_BASE_IMAGE: ubuntu-18-x64 RUN: ${{ github.run_id }}-${{ github.run_number }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + CRT_CI_ROLE: arn:aws:iam::123124136734:role/CRT_CI_Role + AWS_DEFAULT_REGION: us-east-1 AWS_REGION: us-east-1 jobs: @@ -142,6 +141,11 @@ jobs: - gcc-8 steps: # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh From d89746cf95e1e5667fa0b9f6cfb0bbea996588e5 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 09:47:53 -0700 Subject: [PATCH 07/27] fix typo --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5906be60..45544850 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,11 +141,11 @@ jobs: - gcc-8 steps: # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh From dce2f067305081efe95439b68d143c87ce280068 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 09:56:19 -0700 Subject: [PATCH 08/27] id token --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45544850..58d82d27 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,6 @@ env: LINUX_BASE_IMAGE: ubuntu-18-x64 RUN: ${{ github.run_id }}-${{ github.run_number }} CRT_CI_ROLE: arn:aws:iam::123124136734:role/CRT_CI_Role - AWS_DEFAULT_REGION: us-east-1 AWS_REGION: us-east-1 jobs: @@ -116,6 +115,8 @@ jobs: - fedora-34-x64 - opensuse-leap - rhel8-x64 + permissions: + id-token: write # This is required for requesting the JWT steps: # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: Build ${{ env.PACKAGE_NAME }} @@ -145,7 +146,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_DEFAULT_REGION }} + aws-region: ${{ env.AWS_REGION }} - name: Build ${{ env.PACKAGE_NAME }} run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh From 2555dbd08bc4c4eead3db1cad33686781cab1ab2 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 09:58:28 -0700 Subject: [PATCH 09/27] fix ci --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 58d82d27..533f3207 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,8 +115,6 @@ jobs: - fedora-34-x64 - opensuse-leap - rhel8-x64 - permissions: - id-token: write # This is required for requesting the JWT steps: # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: Build ${{ env.PACKAGE_NAME }} @@ -140,6 +138,8 @@ jobs: - gcc-6 - gcc-7 - gcc-8 + permissions: + id-token: write # This is required for requesting the JWT steps: # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) From 45119b1cd7883c837c30f832907d873cdb4754bd Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 10:23:01 -0700 Subject: [PATCH 10/27] copy paste ci fix --- .github/workflows/ci.yml | 201 ++++++++++++++++++++++++++++----------- 1 file changed, 146 insertions(+), 55 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 533f3207..709e79c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ env: PACKAGE_NAME: aws-crt-python LINUX_BASE_IMAGE: ubuntu-18-x64 RUN: ${{ github.run_id }}-${{ github.run_number }} - CRT_CI_ROLE: arn:aws:iam::123124136734:role/CRT_CI_Role + CRT_CI_ROLE: ${{ secrets.CRT_CI_ROLE_ARN }} AWS_REGION: us-east-1 jobs: @@ -29,11 +29,19 @@ jobs: - cp37-cp37m - cp38-cp38 - cp39-cp39 + permissions: + id-token: write # This is required for requesting the JWT steps: - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python manylinux2014: runs-on: ubuntu-20.04 # latest @@ -52,15 +60,23 @@ jobs: - cp311-cp311 - cp312-cp312 - cp313-cp313 + permissions: + id-token: write # This is required for requesting the JWT steps: - # Only aarch64 needs this, but it doesn't hurt anything - - name: Install qemu/docker - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_REGION }} + # Only aarch64 needs this, but it doesn't hurt anything + - name: Install qemu/docker + run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python musllinux-1-1: runs-on: ubuntu-22.04 # latest @@ -78,15 +94,24 @@ jobs: - cp311-cp311 - cp312-cp312 - cp313-cp313 + permissions: + id-token: write # This is required for requesting the JWT steps: - # Only aarch64 needs this, but it doesn't hurt anything - - name: Install qemu/docker - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_REGION }} - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-musllinux-1-1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python + # Only aarch64 needs this, but it doesn't hurt anything + - name: Install qemu/docker + run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-musllinux-1-1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python raspberry: runs-on: ubuntu-20.04 # latest @@ -95,16 +120,24 @@ jobs: matrix: image: - raspbian-bullseye + permissions: + id-token: write # This is required for requesting the JWT steps: - # set arm arch - - name: Install qemu/docker - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_REGION }} - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} + # set arm arch + - name: Install qemu/docker + run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} linux-compat: runs-on: ubuntu-22.04 # latest @@ -115,13 +148,20 @@ jobs: - fedora-34-x64 - opensuse-leap - rhel8-x64 + permissions: + id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} - + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_REGION }} + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} linux-compiler-compat: runs-on: ubuntu-22.04 # latest @@ -154,7 +194,15 @@ jobs: use-system-libcrypto: runs-on: ubuntu-20.04 # latest + permissions: + id-token: write # This is required for requesting the JWT steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_REGION }} - name: Build ${{ env.PACKAGE_NAME }} env: AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO: '1' @@ -180,30 +228,54 @@ jobs: strategy: matrix: arch: [x86, x64] + permissions: + id-token: write # This is required for requesting the JWT steps: - - name: Build ${{ env.PACKAGE_NAME }} + consumers - run: | - python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" - python builder.pyz build -p ${{ env.PACKAGE_NAME }} --python "C:\\hostedtoolcache\\windows\\Python\\3.7.9\\${{ matrix.arch }}\\python.exe" + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" + python builder.pyz build -p ${{ env.PACKAGE_NAME }} --python "C:\\hostedtoolcache\\windows\\Python\\3.7.9\\${{ matrix.arch }}\\python.exe" macos: runs-on: macos-14 # latest + permissions: + id-token: write # This is required for requesting the JWT steps: - - name: Build ${{ env.PACKAGE_NAME }} + consumers - run: | - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} macos-x64: runs-on: macos-14-large # latest + permissions: + id-token: write # This is required for requesting the JWT steps: - - name: Build ${{ env.PACKAGE_NAME }} + consumers - run: | - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} openbsd: @@ -213,7 +285,15 @@ jobs: matrix: # OpenBSD only supports the two most recent releases version: ['7.4', '7.5'] + permissions: + id-token: write # This is required for requesting the JWT steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_REGION }} # Cannot use builder to checkout as OpenBSD doesn't ship git in the base install - uses: actions/checkout@v4 with: @@ -225,7 +305,7 @@ jobs: version: ${{ matrix.version }} cpu_count: 4 shell: bash - environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION AWS_REGION + environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_REGION run: | sudo pkg_add awscli py3-pip py3-urllib3 python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" @@ -234,7 +314,15 @@ jobs: freebsd: runs-on: ubuntu-22.04 # latest + permissions: + id-token: write # This is required for requesting the JWT steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_REGION }} - uses: actions/checkout@v4 with: submodules: true @@ -246,7 +334,7 @@ jobs: version: '14.0' cpu_count: 4 shell: bash - environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION AWS_REGION + environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_REGION run: | sudo pkg install -y python3 devel/py-pip net/py-urllib3 devel/py-awscli cmake python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" @@ -254,18 +342,13 @@ jobs: ./builder build -p ${{ env.PACKAGE_NAME }} # check that tests requiring custom env-vars or AWS credentials are simply skipped - tests-ok-without-env-vars: + tests-ok-without-creds: runs-on: ubuntu-22.04 # latest steps: - uses: actions/checkout@v4 with: submodules: true - - name: Run tests without env-vars or AWS creds - env: - # unset env-vars that provide AWS credentials - AWS_ACCESS_KEY_ID: - AWS_SECRET_ACCESS_KEY: - AWS_DEFAULT_REGION: + - name: Run tests run: | python3 -m pip install --upgrade --requirement requirements-dev.txt python3 -m pip install . --verbose @@ -273,7 +356,15 @@ jobs: package-source: runs-on: ubuntu-22.04 # latest + permissions: + id-token: write # This is required for requesting the JWT steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_REGION }} - uses: actions/checkout@v4 with: submodules: true From f6abe3d4d634d596aa5e5bc655539575068ed5dc Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 10:38:15 -0700 Subject: [PATCH 11/27] test that I have permissions --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 709e79c8..5375f254 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -182,6 +182,10 @@ jobs: id-token: write # This is required for requesting the JWT steps: # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: tests + run: | + aws sts get-caller-identity + aws --region us-east-1 secretsmanager get-secret-value --secret-id unit-test/endpoint - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -305,7 +309,7 @@ jobs: version: ${{ matrix.version }} cpu_count: 4 shell: bash - environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_REGION + environment_variables: AWS_REGION run: | sudo pkg_add awscli py3-pip py3-urllib3 python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" From 7ddadf40f569b8135482bfc0369981862f0549ae Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 10:39:05 -0700 Subject: [PATCH 12/27] stupid me --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5375f254..a12bd8c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -182,15 +182,15 @@ jobs: id-token: write # This is required for requesting the JWT steps: # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: tests - run: | - aws sts get-caller-identity - aws --region us-east-1 secretsmanager get-secret-value --secret-id unit-test/endpoint - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ env.CRT_CI_ROLE }} aws-region: ${{ env.AWS_REGION }} + - name: tests + run: | + aws sts get-caller-identity + aws --region us-east-1 secretsmanager get-secret-value --secret-id unit-test/endpoint - name: Build ${{ env.PACKAGE_NAME }} run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh From 2266d9d398e92bb57777202007b6a47bc9d06d08 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 10:47:21 -0700 Subject: [PATCH 13/27] maybe something uses default region? --- .github/workflows/ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a12bd8c7..1520c14b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,7 @@ env: RUN: ${{ github.run_id }}-${{ github.run_number }} CRT_CI_ROLE: ${{ secrets.CRT_CI_ROLE_ARN }} AWS_REGION: us-east-1 + AWS_DEFAULT_REGION: us-east-1 jobs: manylinux1: @@ -187,10 +188,6 @@ jobs: with: role-to-assume: ${{ env.CRT_CI_ROLE }} aws-region: ${{ env.AWS_REGION }} - - name: tests - run: | - aws sts get-caller-identity - aws --region us-east-1 secretsmanager get-secret-value --secret-id unit-test/endpoint - name: Build ${{ env.PACKAGE_NAME }} run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh From 2adb9d44be395c7ae598cbd0ae54d5f6975f3a35 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 13:18:17 -0700 Subject: [PATCH 14/27] test-creds --- .github/workflows/ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1520c14b..9e7caec7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,8 +7,8 @@ on: - 'docs' env: - BUILDER_VERSION: v0.9.66 - BUILDER_SOURCE: releases + BUILDER_VERSION: test-creds + BUILDER_SOURCE: channels BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net PACKAGE_NAME: aws-crt-python LINUX_BASE_IMAGE: ubuntu-18-x64 @@ -188,6 +188,9 @@ jobs: with: role-to-assume: ${{ env.CRT_CI_ROLE }} aws-region: ${{ env.AWS_REGION }} + - name: Caller Identity + run: | + aws sts get-caller-identity - name: Build ${{ env.PACKAGE_NAME }} run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh From 87ffaa19c43ed50b2d52afd6ab632b4ad3231600 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 13:19:28 -0700 Subject: [PATCH 15/27] disable other CIs --- .github/workflows/ci.yml | 726 +++++++++++++++++++-------------------- 1 file changed, 363 insertions(+), 363 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e7caec7..4f8b1b2d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,167 +18,167 @@ env: AWS_DEFAULT_REGION: us-east-1 jobs: - manylinux1: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - image: - - x64 - - x86 - python: - - cp37-cp37m - - cp38-cp38 - - cp39-cp39 - permissions: - id-token: write # This is required for requesting the JWT - steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python - - manylinux2014: - runs-on: ubuntu-20.04 # latest - strategy: - fail-fast: false - matrix: - image: - - x64 - - x86 - - aarch64 - python: - - cp37-cp37m - - cp38-cp38 - - cp39-cp39 - - cp310-cp310 - - cp311-cp311 - - cp312-cp312 - - cp313-cp313 - permissions: - id-token: write # This is required for requesting the JWT - steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} - # Only aarch64 needs this, but it doesn't hurt anything - - name: Install qemu/docker - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python - - musllinux-1-1: - runs-on: ubuntu-22.04 # latest - strategy: - fail-fast: false - matrix: - image: - - x64 - - aarch64 - python: - - cp37-cp37m - - cp38-cp38 - - cp39-cp39 - - cp310-cp310 - - cp311-cp311 - - cp312-cp312 - - cp313-cp313 - permissions: - id-token: write # This is required for requesting the JWT - steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} - - # Only aarch64 needs this, but it doesn't hurt anything - - name: Install qemu/docker - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-musllinux-1-1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python - - raspberry: - runs-on: ubuntu-20.04 # latest - strategy: - fail-fast: false - matrix: - image: - - raspbian-bullseye - permissions: - id-token: write # This is required for requesting the JWT - steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} - - # set arm arch - - name: Install qemu/docker - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} - - linux-compat: - runs-on: ubuntu-22.04 # latest - strategy: - matrix: - image: - - al2-x64 - - fedora-34-x64 - - opensuse-leap - - rhel8-x64 - permissions: - id-token: write # This is required for requesting the JWT - steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} - +# manylinux1: +# runs-on: ubuntu-latest +# strategy: +# fail-fast: false +# matrix: +# image: +# - x64 +# - x86 +# python: +# - cp37-cp37m +# - cp38-cp38 +# - cp39-cp39 +# permissions: +# id-token: write # This is required for requesting the JWT +# steps: +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: configure AWS credentials (containers) +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: ${{ env.CRT_CI_ROLE }} +# aws-region: ${{ env.AWS_REGION }} +# - name: Build ${{ env.PACKAGE_NAME }} +# run: | +# aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh +# ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python +# +# manylinux2014: +# runs-on: ubuntu-20.04 # latest +# strategy: +# fail-fast: false +# matrix: +# image: +# - x64 +# - x86 +# - aarch64 +# python: +# - cp37-cp37m +# - cp38-cp38 +# - cp39-cp39 +# - cp310-cp310 +# - cp311-cp311 +# - cp312-cp312 +# - cp313-cp313 +# permissions: +# id-token: write # This is required for requesting the JWT +# steps: +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: configure AWS credentials (containers) +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: ${{ env.CRT_CI_ROLE }} +# aws-region: ${{ env.AWS_REGION }} +# # Only aarch64 needs this, but it doesn't hurt anything +# - name: Install qemu/docker +# run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes +# +# - name: Build ${{ env.PACKAGE_NAME }} +# run: | +# aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh +# ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python +# +# musllinux-1-1: +# runs-on: ubuntu-22.04 # latest +# strategy: +# fail-fast: false +# matrix: +# image: +# - x64 +# - aarch64 +# python: +# - cp37-cp37m +# - cp38-cp38 +# - cp39-cp39 +# - cp310-cp310 +# - cp311-cp311 +# - cp312-cp312 +# - cp313-cp313 +# permissions: +# id-token: write # This is required for requesting the JWT +# steps: +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: configure AWS credentials (containers) +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: ${{ env.CRT_CI_ROLE }} +# aws-region: ${{ env.AWS_REGION }} +# +# # Only aarch64 needs this, but it doesn't hurt anything +# - name: Install qemu/docker +# run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes +# +# - name: Build ${{ env.PACKAGE_NAME }} +# run: | +# aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh +# ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-musllinux-1-1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python +# +# raspberry: +# runs-on: ubuntu-20.04 # latest +# strategy: +# fail-fast: false +# matrix: +# image: +# - raspbian-bullseye +# permissions: +# id-token: write # This is required for requesting the JWT +# steps: +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: configure AWS credentials (containers) +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: ${{ env.CRT_CI_ROLE }} +# aws-region: ${{ env.AWS_REGION }} +# +# # set arm arch +# - name: Install qemu/docker +# run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes +# +# - name: Build ${{ env.PACKAGE_NAME }} +# run: | +# aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh +# ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} +# +# linux-compat: +# runs-on: ubuntu-22.04 # latest +# strategy: +# matrix: +# image: +# - al2-x64 +# - fedora-34-x64 +# - opensuse-leap +# - rhel8-x64 +# permissions: +# id-token: write # This is required for requesting the JWT +# steps: +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: configure AWS credentials (containers) +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: ${{ env.CRT_CI_ROLE }} +# aws-region: ${{ env.AWS_REGION }} +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: Build ${{ env.PACKAGE_NAME }} +# run: | +# aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh +# ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} +# linux-compiler-compat: runs-on: ubuntu-22.04 # latest strategy: matrix: compiler: - - clang-3 - - clang-6 - - clang-8 - - clang-9 - - clang-10 +# - clang-3 +# - clang-6 +# - clang-8 +# - clang-9 +# - clang-10 - clang-11 - - gcc-5 - - gcc-6 - - gcc-7 - - gcc-8 +# - gcc-5 +# - gcc-6 +# - gcc-7 +# - gcc-8 permissions: id-token: write # This is required for requesting the JWT steps: @@ -196,211 +196,211 @@ jobs: aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=${{ matrix.compiler }} - use-system-libcrypto: - runs-on: ubuntu-20.04 # latest - permissions: - id-token: write # This is required for requesting the JWT - steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} - - name: Build ${{ env.PACKAGE_NAME }} - env: - AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO: '1' - run: | - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} - - name: Assert libcrypto.so used - run: | - # assert it's linked against the system's libcrypto.so - AWSCRT_PATH=`aws-crt-python/.venv-builder/bin/python3 -c "import _awscrt; print(_awscrt.__file__)"` - printf "AWSCRT_PATH: $AWSCRT_PATH\n" - - LINKED_AGAINST=`ldd $AWSCRT_PATH` - printf "LINKED AGAINST:\n$LINKED_AGAINST\n" - - USES_LIBCRYPTO_SO=`echo "$LINKED_AGAINST" | grep 'libcrypto*.so' | head -1` - test -n "$USES_LIBCRYPTO_SO" - - - windows: - runs-on: windows-2022 # latest - strategy: - matrix: - arch: [x86, x64] - permissions: - id-token: write # This is required for requesting the JWT - steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} - - name: Build ${{ env.PACKAGE_NAME }} + consumers - run: | - python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" - python builder.pyz build -p ${{ env.PACKAGE_NAME }} --python "C:\\hostedtoolcache\\windows\\Python\\3.7.9\\${{ matrix.arch }}\\python.exe" - - - macos: - runs-on: macos-14 # latest - permissions: - id-token: write # This is required for requesting the JWT - steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} - - name: Build ${{ env.PACKAGE_NAME }} + consumers - run: | - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} - - macos-x64: - runs-on: macos-14-large # latest - permissions: - id-token: write # This is required for requesting the JWT - steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} - - name: Build ${{ env.PACKAGE_NAME }} + consumers - run: | - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} - - - openbsd: - runs-on: ubuntu-22.04 # latest - strategy: - fail-fast: false - matrix: - # OpenBSD only supports the two most recent releases - version: ['7.4', '7.5'] - permissions: - id-token: write # This is required for requesting the JWT - steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} - # Cannot use builder to checkout as OpenBSD doesn't ship git in the base install - - uses: actions/checkout@v4 - with: - submodules: true - - name: Build ${{ env.PACKAGE_NAME }} + consumers - uses: cross-platform-actions/action@v0.24.0 - with: - operating_system: openbsd - version: ${{ matrix.version }} - cpu_count: 4 - shell: bash - environment_variables: AWS_REGION - run: | - sudo pkg_add awscli py3-pip py3-urllib3 - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} - - freebsd: - runs-on: ubuntu-22.04 # latest - permissions: - id-token: write # This is required for requesting the JWT - steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} - - uses: actions/checkout@v4 - with: - submodules: true - - - name: Build ${{ env.PACKAGE_NAME }} + consumers - uses: cross-platform-actions/action@v0.23.0 - with: - operating_system: freebsd - version: '14.0' - cpu_count: 4 - shell: bash - environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_REGION - run: | - sudo pkg install -y python3 devel/py-pip net/py-urllib3 devel/py-awscli cmake - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} - - # check that tests requiring custom env-vars or AWS credentials are simply skipped - tests-ok-without-creds: - runs-on: ubuntu-22.04 # latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Run tests - run: | - python3 -m pip install --upgrade --requirement requirements-dev.txt - python3 -m pip install . --verbose - python3 -m unittest discover --failfast --verbose - - package-source: - runs-on: ubuntu-22.04 # latest - permissions: - id-token: write # This is required for requesting the JWT - steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} - - uses: actions/checkout@v4 - with: - submodules: true - - name: Package source + install - run: | - python3 setup.py sdist - cd dist - python3 -m pip install -v awscrt-1.0.0.dev0.tar.gz - python3 -c "import awscrt.io" - - # check that docs can still build - check-docs: - runs-on: ubuntu-22.04 # latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Check docs - run: | - python3 -m pip install -r requirements-dev.txt - python3 -m pip install --verbose . - ./scripts/make-docs.py - - check-submodules: - runs-on: ubuntu-22.04 # latest - steps: - - name: Checkout Source - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 0 - - name: Check Submodules - # note: using "@main" because "@${{env.BUILDER_VERSION}}" doesn't work - # https://github.com/actions/runner/issues/480 - uses: awslabs/aws-crt-builder/.github/actions/check-submodules@main +# use-system-libcrypto: +# runs-on: ubuntu-20.04 # latest +# permissions: +# id-token: write # This is required for requesting the JWT +# steps: +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: configure AWS credentials (containers) +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: ${{ env.CRT_CI_ROLE }} +# aws-region: ${{ env.AWS_REGION }} +# - name: Build ${{ env.PACKAGE_NAME }} +# env: +# AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO: '1' +# run: | +# python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" +# chmod a+x builder +# ./builder build -p ${{ env.PACKAGE_NAME }} +# - name: Assert libcrypto.so used +# run: | +# # assert it's linked against the system's libcrypto.so +# AWSCRT_PATH=`aws-crt-python/.venv-builder/bin/python3 -c "import _awscrt; print(_awscrt.__file__)"` +# printf "AWSCRT_PATH: $AWSCRT_PATH\n" +# +# LINKED_AGAINST=`ldd $AWSCRT_PATH` +# printf "LINKED AGAINST:\n$LINKED_AGAINST\n" +# +# USES_LIBCRYPTO_SO=`echo "$LINKED_AGAINST" | grep 'libcrypto*.so' | head -1` +# test -n "$USES_LIBCRYPTO_SO" +# +# +# windows: +# runs-on: windows-2022 # latest +# strategy: +# matrix: +# arch: [x86, x64] +# permissions: +# id-token: write # This is required for requesting the JWT +# steps: +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: configure AWS credentials (containers) +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: ${{ env.CRT_CI_ROLE }} +# aws-region: ${{ env.AWS_REGION }} +# - name: Build ${{ env.PACKAGE_NAME }} + consumers +# run: | +# python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" +# python builder.pyz build -p ${{ env.PACKAGE_NAME }} --python "C:\\hostedtoolcache\\windows\\Python\\3.7.9\\${{ matrix.arch }}\\python.exe" +# +# +# macos: +# runs-on: macos-14 # latest +# permissions: +# id-token: write # This is required for requesting the JWT +# steps: +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: configure AWS credentials (containers) +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: ${{ env.CRT_CI_ROLE }} +# aws-region: ${{ env.AWS_REGION }} +# - name: Build ${{ env.PACKAGE_NAME }} + consumers +# run: | +# python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" +# chmod a+x builder +# ./builder build -p ${{ env.PACKAGE_NAME }} +# +# macos-x64: +# runs-on: macos-14-large # latest +# permissions: +# id-token: write # This is required for requesting the JWT +# steps: +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: configure AWS credentials (containers) +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: ${{ env.CRT_CI_ROLE }} +# aws-region: ${{ env.AWS_REGION }} +# - name: Build ${{ env.PACKAGE_NAME }} + consumers +# run: | +# python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" +# chmod a+x builder +# ./builder build -p ${{ env.PACKAGE_NAME }} +# +# +# openbsd: +# runs-on: ubuntu-22.04 # latest +# strategy: +# fail-fast: false +# matrix: +# # OpenBSD only supports the two most recent releases +# version: ['7.4', '7.5'] +# permissions: +# id-token: write # This is required for requesting the JWT +# steps: +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: configure AWS credentials (containers) +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: ${{ env.CRT_CI_ROLE }} +# aws-region: ${{ env.AWS_REGION }} +# # Cannot use builder to checkout as OpenBSD doesn't ship git in the base install +# - uses: actions/checkout@v4 +# with: +# submodules: true +# - name: Build ${{ env.PACKAGE_NAME }} + consumers +# uses: cross-platform-actions/action@v0.24.0 +# with: +# operating_system: openbsd +# version: ${{ matrix.version }} +# cpu_count: 4 +# shell: bash +# environment_variables: AWS_REGION +# run: | +# sudo pkg_add awscli py3-pip py3-urllib3 +# python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" +# chmod a+x builder +# ./builder build -p ${{ env.PACKAGE_NAME }} +# +# freebsd: +# runs-on: ubuntu-22.04 # latest +# permissions: +# id-token: write # This is required for requesting the JWT +# steps: +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: configure AWS credentials (containers) +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: ${{ env.CRT_CI_ROLE }} +# aws-region: ${{ env.AWS_REGION }} +# - uses: actions/checkout@v4 +# with: +# submodules: true +# +# - name: Build ${{ env.PACKAGE_NAME }} + consumers +# uses: cross-platform-actions/action@v0.23.0 +# with: +# operating_system: freebsd +# version: '14.0' +# cpu_count: 4 +# shell: bash +# environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_REGION +# run: | +# sudo pkg install -y python3 devel/py-pip net/py-urllib3 devel/py-awscli cmake +# python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" +# chmod a+x builder +# ./builder build -p ${{ env.PACKAGE_NAME }} +# +# # check that tests requiring custom env-vars or AWS credentials are simply skipped +# tests-ok-without-creds: +# runs-on: ubuntu-22.04 # latest +# steps: +# - uses: actions/checkout@v4 +# with: +# submodules: true +# - name: Run tests +# run: | +# python3 -m pip install --upgrade --requirement requirements-dev.txt +# python3 -m pip install . --verbose +# python3 -m unittest discover --failfast --verbose +# +# package-source: +# runs-on: ubuntu-22.04 # latest +# permissions: +# id-token: write # This is required for requesting the JWT +# steps: +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: configure AWS credentials (containers) +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: ${{ env.CRT_CI_ROLE }} +# aws-region: ${{ env.AWS_REGION }} +# - uses: actions/checkout@v4 +# with: +# submodules: true +# - name: Package source + install +# run: | +# python3 setup.py sdist +# cd dist +# python3 -m pip install -v awscrt-1.0.0.dev0.tar.gz +# python3 -c "import awscrt.io" +# +# # check that docs can still build +# check-docs: +# runs-on: ubuntu-22.04 # latest +# steps: +# - uses: actions/checkout@v4 +# with: +# submodules: true +# - name: Check docs +# run: | +# python3 -m pip install -r requirements-dev.txt +# python3 -m pip install --verbose . +# ./scripts/make-docs.py +# +# check-submodules: +# runs-on: ubuntu-22.04 # latest +# steps: +# - name: Checkout Source +# uses: actions/checkout@v4 +# with: +# submodules: true +# fetch-depth: 0 +# - name: Check Submodules +# # note: using "@main" because "@${{env.BUILDER_VERSION}}" doesn't work +# # https://github.com/actions/runner/issues/480 +# uses: awslabs/aws-crt-builder/.github/actions/check-submodules@main From f820bab41b53acc0f0b535b1ffdfee4350146dd2 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 14:05:42 -0700 Subject: [PATCH 16/27] fix unused warning --- source/mqtt_client_connection.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/mqtt_client_connection.c b/source/mqtt_client_connection.c index ab5c5291..ee6496ac 100644 --- a/source/mqtt_client_connection.c +++ b/source/mqtt_client_connection.c @@ -160,6 +160,7 @@ static void s_on_connection_success( PyErr_WriteUnraisable(PyErr_Occurred()); } } + goto on_done; /* fixes unused label waring */ on_done: #if PY_VERSION_HEX >= 0x030D0000 Py_XDECREF(self); @@ -200,6 +201,7 @@ static void s_on_connection_failure(struct aws_mqtt_client_connection *connectio } } + goto on_done; /* fixes unused label waring */ on_done: #if PY_VERSION_HEX >= 0x030D0000 Py_XDECREF(self); @@ -241,6 +243,7 @@ static void s_on_connection_interrupted(struct aws_mqtt_client_connection *conne } } + goto on_done; /* fixes unused label waring */ on_done: #if PY_VERSION_HEX >= 0x030D0000 Py_XDECREF(self); @@ -288,6 +291,7 @@ static void s_on_connection_resumed( PyErr_WriteUnraisable(PyErr_Occurred()); } } + goto on_done; /* fixes unused label waring */ on_done: #if PY_VERSION_HEX >= 0x030D0000 Py_XDECREF(self); @@ -332,6 +336,7 @@ static void s_on_connection_closed( } } + goto on_done; /* fixes unused label waring */ on_done: #if PY_VERSION_HEX >= 0x030D0000 Py_XDECREF(self); From 281d3c69f94c6400948e2f2a989b0c260ae5dd84 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 14:21:03 -0700 Subject: [PATCH 17/27] fix bugs --- source/http_stream.c | 5 +++-- source/mqtt_client_connection.c | 12 ++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/source/http_stream.c b/source/http_stream.c index f6f09bde..bcf0f948 100644 --- a/source/http_stream.c +++ b/source/http_stream.c @@ -205,11 +205,12 @@ static void s_on_stream_complete(struct aws_http_stream *native_stream, int erro /* DECREF python self, we don't need to force it to stay alive any longer. */ PyObject *self = Py_None; #if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ + int result_getref = PyWeakref_GetRef(stream->self_proxy, &self);/* strong reference */ /* Ignore error, stream is already complete */ - PyWeakref_GetRef(stream->self_proxy, &self) < 0);/* strong reference */ + (void) result_getref; #else /* PyWeakref_GetObject is deprecated since python 3.13 */ - PyWeakref_GetObject(stream->self_proxy); /* borrowed reference */ + self = PyWeakref_GetObject(stream->self_proxy); /* borrowed reference */ #endif Py_XDECREF(self); #if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ diff --git a/source/mqtt_client_connection.c b/source/mqtt_client_connection.c index ee6496ac..513604d9 100644 --- a/source/mqtt_client_connection.c +++ b/source/mqtt_client_connection.c @@ -148,7 +148,7 @@ static void s_on_connection_success( } #else /* PyWeakref_GetObject is deprecated since python 3.13 */ - PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ #endif if (self != Py_None) { @@ -189,7 +189,7 @@ static void s_on_connection_failure(struct aws_mqtt_client_connection *connectio } #else /* PyWeakref_GetObject is deprecated since python 3.13 */ - PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ #endif if (self != Py_None) { @@ -231,7 +231,7 @@ static void s_on_connection_interrupted(struct aws_mqtt_client_connection *conne } #else /* PyWeakref_GetObject is deprecated since python 3.13 */ - PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ #endif if (self != Py_None) { @@ -279,7 +279,7 @@ static void s_on_connection_resumed( } #else /* PyWeakref_GetObject is deprecated since python 3.13 */ - PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ #endif if (self != Py_None) { @@ -324,7 +324,7 @@ static void s_on_connection_closed( } #else /* PyWeakref_GetObject is deprecated since python 3.13 */ - PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ #endif if (self != Py_None) { @@ -616,7 +616,7 @@ static void s_ws_handshake_transform( } #else /* PyWeakref_GetObject is deprecated since python 3.13 */ - PyWeakref_GetObject(connection_binding->self_proxy); /* borrowed reference */ + connection_py = PyWeakref_GetObject(connection_binding->self_proxy); /* borrowed reference */ #endif if (connection_py == Py_None) { From e94ed91b44f1876715393acbbe3fcb9170f347a3 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 14:25:58 -0700 Subject: [PATCH 18/27] reenable CI --- .github/workflows/ci.yml | 729 +++++++++++++++++++-------------------- 1 file changed, 364 insertions(+), 365 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f8b1b2d..e5369d82 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,171 +14,170 @@ env: LINUX_BASE_IMAGE: ubuntu-18-x64 RUN: ${{ github.run_id }}-${{ github.run_number }} CRT_CI_ROLE: ${{ secrets.CRT_CI_ROLE_ARN }} - AWS_REGION: us-east-1 AWS_DEFAULT_REGION: us-east-1 jobs: -# manylinux1: -# runs-on: ubuntu-latest -# strategy: -# fail-fast: false -# matrix: -# image: -# - x64 -# - x86 -# python: -# - cp37-cp37m -# - cp38-cp38 -# - cp39-cp39 -# permissions: -# id-token: write # This is required for requesting the JWT -# steps: -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: configure AWS credentials (containers) -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# role-to-assume: ${{ env.CRT_CI_ROLE }} -# aws-region: ${{ env.AWS_REGION }} -# - name: Build ${{ env.PACKAGE_NAME }} -# run: | -# aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh -# ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python -# -# manylinux2014: -# runs-on: ubuntu-20.04 # latest -# strategy: -# fail-fast: false -# matrix: -# image: -# - x64 -# - x86 -# - aarch64 -# python: -# - cp37-cp37m -# - cp38-cp38 -# - cp39-cp39 -# - cp310-cp310 -# - cp311-cp311 -# - cp312-cp312 -# - cp313-cp313 -# permissions: -# id-token: write # This is required for requesting the JWT -# steps: -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: configure AWS credentials (containers) -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# role-to-assume: ${{ env.CRT_CI_ROLE }} -# aws-region: ${{ env.AWS_REGION }} -# # Only aarch64 needs this, but it doesn't hurt anything -# - name: Install qemu/docker -# run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -# -# - name: Build ${{ env.PACKAGE_NAME }} -# run: | -# aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh -# ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python -# -# musllinux-1-1: -# runs-on: ubuntu-22.04 # latest -# strategy: -# fail-fast: false -# matrix: -# image: -# - x64 -# - aarch64 -# python: -# - cp37-cp37m -# - cp38-cp38 -# - cp39-cp39 -# - cp310-cp310 -# - cp311-cp311 -# - cp312-cp312 -# - cp313-cp313 -# permissions: -# id-token: write # This is required for requesting the JWT -# steps: -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: configure AWS credentials (containers) -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# role-to-assume: ${{ env.CRT_CI_ROLE }} -# aws-region: ${{ env.AWS_REGION }} -# -# # Only aarch64 needs this, but it doesn't hurt anything -# - name: Install qemu/docker -# run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -# -# - name: Build ${{ env.PACKAGE_NAME }} -# run: | -# aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh -# ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-musllinux-1-1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python -# -# raspberry: -# runs-on: ubuntu-20.04 # latest -# strategy: -# fail-fast: false -# matrix: -# image: -# - raspbian-bullseye -# permissions: -# id-token: write # This is required for requesting the JWT -# steps: -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: configure AWS credentials (containers) -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# role-to-assume: ${{ env.CRT_CI_ROLE }} -# aws-region: ${{ env.AWS_REGION }} -# -# # set arm arch -# - name: Install qemu/docker -# run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -# -# - name: Build ${{ env.PACKAGE_NAME }} -# run: | -# aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh -# ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} -# -# linux-compat: -# runs-on: ubuntu-22.04 # latest -# strategy: -# matrix: -# image: -# - al2-x64 -# - fedora-34-x64 -# - opensuse-leap -# - rhel8-x64 -# permissions: -# id-token: write # This is required for requesting the JWT -# steps: -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: configure AWS credentials (containers) -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# role-to-assume: ${{ env.CRT_CI_ROLE }} -# aws-region: ${{ env.AWS_REGION }} -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: Build ${{ env.PACKAGE_NAME }} -# run: | -# aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh -# ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} -# + manylinux1: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + image: + - x64 + - x86 + python: + - cp37-cp37m + - cp38-cp38 + - cp39-cp39 + permissions: + id-token: write # This is required for requesting the JWT + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python + + manylinux2014: + runs-on: ubuntu-20.04 # latest + strategy: + fail-fast: false + matrix: + image: + - x64 + - x86 + - aarch64 + python: + - cp37-cp37m + - cp38-cp38 + - cp39-cp39 + - cp310-cp310 + - cp311-cp311 + - cp312-cp312 + - cp313-cp313 + permissions: + id-token: write # This is required for requesting the JWT + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + # Only aarch64 needs this, but it doesn't hurt anything + - name: Install qemu/docker + run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python + + musllinux-1-1: + runs-on: ubuntu-22.04 # latest + strategy: + fail-fast: false + matrix: + image: + - x64 + - aarch64 + python: + - cp37-cp37m + - cp38-cp38 + - cp39-cp39 + - cp310-cp310 + - cp311-cp311 + - cp312-cp312 + - cp313-cp313 + permissions: + id-token: write # This is required for requesting the JWT + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + + # Only aarch64 needs this, but it doesn't hurt anything + - name: Install qemu/docker + run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-musllinux-1-1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python + + raspberry: + runs-on: ubuntu-20.04 # latest + strategy: + fail-fast: false + matrix: + image: + - raspbian-bullseye + permissions: + id-token: write # This is required for requesting the JWT + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + + # set arm arch + - name: Install qemu/docker + run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} + + linux-compat: + runs-on: ubuntu-22.04 # latest + strategy: + matrix: + image: + - al2-x64 + - fedora-34-x64 + - opensuse-leap + - rhel8-x64 + permissions: + id-token: write # This is required for requesting the JWT + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} + linux-compiler-compat: runs-on: ubuntu-22.04 # latest strategy: matrix: compiler: -# - clang-3 -# - clang-6 -# - clang-8 -# - clang-9 -# - clang-10 + - clang-3 + - clang-6 + - clang-8 + - clang-9 + - clang-10 - clang-11 -# - gcc-5 -# - gcc-6 -# - gcc-7 -# - gcc-8 + - gcc-5 + - gcc-6 + - gcc-7 + - gcc-8 permissions: id-token: write # This is required for requesting the JWT steps: @@ -187,7 +186,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Caller Identity run: | aws sts get-caller-identity @@ -196,211 +195,211 @@ jobs: aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=${{ matrix.compiler }} -# use-system-libcrypto: -# runs-on: ubuntu-20.04 # latest -# permissions: -# id-token: write # This is required for requesting the JWT -# steps: -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: configure AWS credentials (containers) -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# role-to-assume: ${{ env.CRT_CI_ROLE }} -# aws-region: ${{ env.AWS_REGION }} -# - name: Build ${{ env.PACKAGE_NAME }} -# env: -# AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO: '1' -# run: | -# python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" -# chmod a+x builder -# ./builder build -p ${{ env.PACKAGE_NAME }} -# - name: Assert libcrypto.so used -# run: | -# # assert it's linked against the system's libcrypto.so -# AWSCRT_PATH=`aws-crt-python/.venv-builder/bin/python3 -c "import _awscrt; print(_awscrt.__file__)"` -# printf "AWSCRT_PATH: $AWSCRT_PATH\n" -# -# LINKED_AGAINST=`ldd $AWSCRT_PATH` -# printf "LINKED AGAINST:\n$LINKED_AGAINST\n" -# -# USES_LIBCRYPTO_SO=`echo "$LINKED_AGAINST" | grep 'libcrypto*.so' | head -1` -# test -n "$USES_LIBCRYPTO_SO" -# -# -# windows: -# runs-on: windows-2022 # latest -# strategy: -# matrix: -# arch: [x86, x64] -# permissions: -# id-token: write # This is required for requesting the JWT -# steps: -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: configure AWS credentials (containers) -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# role-to-assume: ${{ env.CRT_CI_ROLE }} -# aws-region: ${{ env.AWS_REGION }} -# - name: Build ${{ env.PACKAGE_NAME }} + consumers -# run: | -# python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" -# python builder.pyz build -p ${{ env.PACKAGE_NAME }} --python "C:\\hostedtoolcache\\windows\\Python\\3.7.9\\${{ matrix.arch }}\\python.exe" -# -# -# macos: -# runs-on: macos-14 # latest -# permissions: -# id-token: write # This is required for requesting the JWT -# steps: -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: configure AWS credentials (containers) -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# role-to-assume: ${{ env.CRT_CI_ROLE }} -# aws-region: ${{ env.AWS_REGION }} -# - name: Build ${{ env.PACKAGE_NAME }} + consumers -# run: | -# python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" -# chmod a+x builder -# ./builder build -p ${{ env.PACKAGE_NAME }} -# -# macos-x64: -# runs-on: macos-14-large # latest -# permissions: -# id-token: write # This is required for requesting the JWT -# steps: -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: configure AWS credentials (containers) -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# role-to-assume: ${{ env.CRT_CI_ROLE }} -# aws-region: ${{ env.AWS_REGION }} -# - name: Build ${{ env.PACKAGE_NAME }} + consumers -# run: | -# python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" -# chmod a+x builder -# ./builder build -p ${{ env.PACKAGE_NAME }} -# -# -# openbsd: -# runs-on: ubuntu-22.04 # latest -# strategy: -# fail-fast: false -# matrix: -# # OpenBSD only supports the two most recent releases -# version: ['7.4', '7.5'] -# permissions: -# id-token: write # This is required for requesting the JWT -# steps: -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: configure AWS credentials (containers) -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# role-to-assume: ${{ env.CRT_CI_ROLE }} -# aws-region: ${{ env.AWS_REGION }} -# # Cannot use builder to checkout as OpenBSD doesn't ship git in the base install -# - uses: actions/checkout@v4 -# with: -# submodules: true -# - name: Build ${{ env.PACKAGE_NAME }} + consumers -# uses: cross-platform-actions/action@v0.24.0 -# with: -# operating_system: openbsd -# version: ${{ matrix.version }} -# cpu_count: 4 -# shell: bash -# environment_variables: AWS_REGION -# run: | -# sudo pkg_add awscli py3-pip py3-urllib3 -# python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" -# chmod a+x builder -# ./builder build -p ${{ env.PACKAGE_NAME }} -# -# freebsd: -# runs-on: ubuntu-22.04 # latest -# permissions: -# id-token: write # This is required for requesting the JWT -# steps: -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: configure AWS credentials (containers) -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# role-to-assume: ${{ env.CRT_CI_ROLE }} -# aws-region: ${{ env.AWS_REGION }} -# - uses: actions/checkout@v4 -# with: -# submodules: true -# -# - name: Build ${{ env.PACKAGE_NAME }} + consumers -# uses: cross-platform-actions/action@v0.23.0 -# with: -# operating_system: freebsd -# version: '14.0' -# cpu_count: 4 -# shell: bash -# environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_REGION -# run: | -# sudo pkg install -y python3 devel/py-pip net/py-urllib3 devel/py-awscli cmake -# python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" -# chmod a+x builder -# ./builder build -p ${{ env.PACKAGE_NAME }} -# -# # check that tests requiring custom env-vars or AWS credentials are simply skipped -# tests-ok-without-creds: -# runs-on: ubuntu-22.04 # latest -# steps: -# - uses: actions/checkout@v4 -# with: -# submodules: true -# - name: Run tests -# run: | -# python3 -m pip install --upgrade --requirement requirements-dev.txt -# python3 -m pip install . --verbose -# python3 -m unittest discover --failfast --verbose -# -# package-source: -# runs-on: ubuntu-22.04 # latest -# permissions: -# id-token: write # This is required for requesting the JWT -# steps: -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: configure AWS credentials (containers) -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# role-to-assume: ${{ env.CRT_CI_ROLE }} -# aws-region: ${{ env.AWS_REGION }} -# - uses: actions/checkout@v4 -# with: -# submodules: true -# - name: Package source + install -# run: | -# python3 setup.py sdist -# cd dist -# python3 -m pip install -v awscrt-1.0.0.dev0.tar.gz -# python3 -c "import awscrt.io" -# -# # check that docs can still build -# check-docs: -# runs-on: ubuntu-22.04 # latest -# steps: -# - uses: actions/checkout@v4 -# with: -# submodules: true -# - name: Check docs -# run: | -# python3 -m pip install -r requirements-dev.txt -# python3 -m pip install --verbose . -# ./scripts/make-docs.py -# -# check-submodules: -# runs-on: ubuntu-22.04 # latest -# steps: -# - name: Checkout Source -# uses: actions/checkout@v4 -# with: -# submodules: true -# fetch-depth: 0 -# - name: Check Submodules -# # note: using "@main" because "@${{env.BUILDER_VERSION}}" doesn't work -# # https://github.com/actions/runner/issues/480 -# uses: awslabs/aws-crt-builder/.github/actions/check-submodules@main + use-system-libcrypto: + runs-on: ubuntu-20.04 # latest + permissions: + id-token: write # This is required for requesting the JWT + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + env: + AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO: '1' + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} + - name: Assert libcrypto.so used + run: | + # assert it's linked against the system's libcrypto.so + AWSCRT_PATH=`aws-crt-python/.venv-builder/bin/python3 -c "import _awscrt; print(_awscrt.__file__)"` + printf "AWSCRT_PATH: $AWSCRT_PATH\n" + + LINKED_AGAINST=`ldd $AWSCRT_PATH` + printf "LINKED AGAINST:\n$LINKED_AGAINST\n" + + USES_LIBCRYPTO_SO=`echo "$LINKED_AGAINST" | grep 'libcrypto*.so' | head -1` + test -n "$USES_LIBCRYPTO_SO" + + + windows: + runs-on: windows-2022 # latest + strategy: + matrix: + arch: [x86, x64] + permissions: + id-token: write # This is required for requesting the JWT + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" + python builder.pyz build -p ${{ env.PACKAGE_NAME }} --python "C:\\hostedtoolcache\\windows\\Python\\3.7.9\\${{ matrix.arch }}\\python.exe" + + + macos: + runs-on: macos-14 # latest + permissions: + id-token: write # This is required for requesting the JWT + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} + + macos-x64: + runs-on: macos-14-large # latest + permissions: + id-token: write # This is required for requesting the JWT + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} + + + openbsd: + runs-on: ubuntu-22.04 # latest + strategy: + fail-fast: false + matrix: + # OpenBSD only supports the two most recent releases + version: ['7.4', '7.5'] + permissions: + id-token: write # This is required for requesting the JWT + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + # Cannot use builder to checkout as OpenBSD doesn't ship git in the base install + - uses: actions/checkout@v4 + with: + submodules: true + - name: Build ${{ env.PACKAGE_NAME }} + consumers + uses: cross-platform-actions/action@v0.24.0 + with: + operating_system: openbsd + version: ${{ matrix.version }} + cpu_count: 4 + shell: bash + environment_variables: AWS_DEFAULT_REGION + run: | + sudo pkg_add awscli py3-pip py3-urllib3 + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} + + freebsd: + runs-on: ubuntu-22.04 # latest + permissions: + id-token: write # This is required for requesting the JWT + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - uses: actions/checkout@v4 + with: + submodules: true + + - name: Build ${{ env.PACKAGE_NAME }} + consumers + uses: cross-platform-actions/action@v0.23.0 + with: + operating_system: freebsd + version: '14.0' + cpu_count: 4 + shell: bash + environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION AWS_REGION + run: | + sudo pkg install -y python3 devel/py-pip net/py-urllib3 devel/py-awscli cmake + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} + + # check that tests requiring custom env-vars or AWS credentials are simply skipped + tests-ok-without-creds: + runs-on: ubuntu-22.04 # latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Run tests + run: | + python3 -m pip install --upgrade --requirement requirements-dev.txt + python3 -m pip install . --verbose + python3 -m unittest discover --failfast --verbose + + package-source: + runs-on: ubuntu-22.04 # latest + permissions: + id-token: write # This is required for requesting the JWT + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - uses: actions/checkout@v4 + with: + submodules: true + - name: Package source + install + run: | + python3 setup.py sdist + cd dist + python3 -m pip install -v awscrt-1.0.0.dev0.tar.gz + python3 -c "import awscrt.io" + + # check that docs can still build + check-docs: + runs-on: ubuntu-22.04 # latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Check docs + run: | + python3 -m pip install -r requirements-dev.txt + python3 -m pip install --verbose . + ./scripts/make-docs.py + + check-submodules: + runs-on: ubuntu-22.04 # latest + steps: + - name: Checkout Source + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 0 + - name: Check Submodules + # note: using "@main" because "@${{env.BUILDER_VERSION}}" doesn't work + # https://github.com/actions/runner/issues/480 + uses: awslabs/aws-crt-builder/.github/actions/check-submodules@main From c5ff663600c196f1bb614a60c5da7473dbb412aa Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 14:26:46 -0700 Subject: [PATCH 19/27] lint --- setup.py | 4 ++-- source/http_stream.c | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index cece200c..a14239bd 100644 --- a/setup.py +++ b/setup.py @@ -403,10 +403,10 @@ def awscrt_ext(): else: extra_link_args += ['-Wl,--fatal-warnings'] - if sys.version_info >= (3,13): + if sys.version_info >= (3, 13): define_macros.append(('Py_LIMITED_API', '0x030D0000')) py_limited_api = True - elif sys.version_info >= (3,11): + elif sys.version_info >= (3, 11): define_macros.append(('Py_LIMITED_API', '0x030B0000')) py_limited_api = True diff --git a/source/http_stream.c b/source/http_stream.c index bcf0f948..5097771e 100644 --- a/source/http_stream.c +++ b/source/http_stream.c @@ -204,10 +204,10 @@ static void s_on_stream_complete(struct aws_http_stream *native_stream, int erro /* DECREF python self, we don't need to force it to stay alive any longer. */ PyObject *self = Py_None; -#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ - int result_getref = PyWeakref_GetRef(stream->self_proxy, &self);/* strong reference */ +#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ + int result_getref = PyWeakref_GetRef(stream->self_proxy, &self); /* strong reference */ /* Ignore error, stream is already complete */ - (void) result_getref; + (void)result_getref; #else /* PyWeakref_GetObject is deprecated since python 3.13 */ self = PyWeakref_GetObject(stream->self_proxy); /* borrowed reference */ From 3577f753171e081864639660c96b518c314ed2df Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 14:30:37 -0700 Subject: [PATCH 20/27] fix freebsd --- .github/workflows/ci.yml | 2 +- continuous-delivery/build-wheels-manylinux2014-aarch64.sh | 2 +- continuous-delivery/build-wheels-manylinux2014-x86_64.sh | 2 +- continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh | 2 +- continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh | 2 +- continuous-delivery/build-wheels-osx.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5369d82..abccae40 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -337,7 +337,7 @@ jobs: version: '14.0' cpu_count: 4 shell: bash - environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION AWS_REGION + environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN AWS_DEFAULT_REGION run: | sudo pkg install -y python3 devel/py-pip net/py-urllib3 devel/py-awscli cmake python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" diff --git a/continuous-delivery/build-wheels-manylinux2014-aarch64.sh b/continuous-delivery/build-wheels-manylinux2014-aarch64.sh index 8ff18c89..3deed7ff 100755 --- a/continuous-delivery/build-wheels-manylinux2014-aarch64.sh +++ b/continuous-delivery/build-wheels-manylinux2014-aarch64.sh @@ -21,7 +21,7 @@ auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp311*.whl # Don't need to build wheels for Python 3.12 and later. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. -# We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions. /opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp313*.whl diff --git a/continuous-delivery/build-wheels-manylinux2014-x86_64.sh b/continuous-delivery/build-wheels-manylinux2014-x86_64.sh index 2714fe9e..0b66d34e 100755 --- a/continuous-delivery/build-wheels-manylinux2014-x86_64.sh +++ b/continuous-delivery/build-wheels-manylinux2014-x86_64.sh @@ -21,7 +21,7 @@ auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp311*.whl # Don't need to build wheels for Python 3.12 and later. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. -# We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions. /opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp313*.whl diff --git a/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh b/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh index c2f13cff..add3a38b 100755 --- a/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh +++ b/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh @@ -22,7 +22,7 @@ auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp311*.whl # Don't need to build wheels for Python 3.12 and later. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. -# We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions. /opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp313*.whl diff --git a/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh b/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh index 250cc5ba..9f405705 100755 --- a/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh +++ b/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh @@ -22,7 +22,7 @@ auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp311*.whl # Don't need to build wheels for Python 3.12 and later. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. -# We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions. /opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp313*.whl diff --git a/continuous-delivery/build-wheels-osx.sh b/continuous-delivery/build-wheels-osx.sh index 1faf2688..802c5d34 100755 --- a/continuous-delivery/build-wheels-osx.sh +++ b/continuous-delivery/build-wheels-osx.sh @@ -14,7 +14,7 @@ set -ex # Don't need to build wheels for Python 3.12 and later. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. -# We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions. /Library/Frameworks/Python.framework/Versions/3.13/bin/python3 setup.py sdist bdist_wheel #now you just need to run twine (that's in a different script) From 5b38b528e17c3fe5b730a8a96c8c896c69491494 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 14:36:39 -0700 Subject: [PATCH 21/27] fix openbsd --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index abccae40..bb7aec11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -308,7 +308,7 @@ jobs: version: ${{ matrix.version }} cpu_count: 4 shell: bash - environment_variables: AWS_DEFAULT_REGION + environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN AWS_DEFAULT_REGION run: | sudo pkg_add awscli py3-pip py3-urllib3 python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" From 0b598b01079aec8785682126ac34e8620b240fa3 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 14:53:51 -0700 Subject: [PATCH 22/27] fix default chain test --- test/test_mqtt5_credentials.py | 44 ++++++++-------------------------- 1 file changed, 10 insertions(+), 34 deletions(-) diff --git a/test/test_mqtt5_credentials.py b/test/test_mqtt5_credentials.py index 03e5c401..b902e2bb 100644 --- a/test/test_mqtt5_credentials.py +++ b/test/test_mqtt5_credentials.py @@ -222,39 +222,6 @@ def sign_function(transform_args, **kwargs): client.stop() callbacks.future_stopped.result(TIMEOUT) - def test_mqtt5_ws_cred_default(self): - input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") - input_region = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_REGION") - - client_options = mqtt5.ClientOptions( - host_name=input_host_name, - port=443 - ) - credentials = auth.AwsCredentialsProvider.new_default_chain() - - def sign_function(transform_args, **kwargs): - signing_config = auth.AwsSigningConfig( - algorithm=auth.AwsSigningAlgorithm.V4, - signature_type=auth.AwsSignatureType.HTTP_REQUEST_QUERY_PARAMS, - credentials_provider=credentials, - region=input_region, - service="iotdevicegateway", - omit_session_token=True - ) - signing_future = auth.aws_sign_request( - http_request=transform_args.http_request, - signing_config=signing_config) - signing_future.add_done_callback(lambda x: transform_args.set_done(x.exception())) - client_options.websocket_handshake_transform = sign_function - client_options.tls_ctx = io.ClientTlsContext(io.TlsContextOptions()) - - callbacks = Mqtt5TestCallbacks() - client = self._create_client(client_options=client_options, callbacks=callbacks) - client.start() - callbacks.future_connection_success.result(TIMEOUT) - client.stop() - callbacks.future_stopped.result(TIMEOUT) - def test_mqtt5_ws_cred_cognito(self): input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") input_cognito_endpoint = _get_env_variable("AWS_TEST_MQTT5_COGNITO_ENDPOINT") @@ -380,6 +347,12 @@ def sign_function(transform_args, **kwargs): callbacks.future_stopped.result(TIMEOUT) def test_mqtt5_ws_cred_environment(self): + self._test_mqtt5_ws_cred_environment(use_default_chain=False) + + def test_mqtt5_ws_cred_default_chain(self): + self._test_mqtt5_ws_cred_environment(use_default_chain=True) + + def _test_mqtt5_ws_cred_environment(self, use_default_chain): input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") input_access_key = _get_env_variable("AWS_TEST_MQTT5_ROLE_CREDENTIAL_ACCESS_KEY") input_secret_access_key = _get_env_variable("AWS_TEST_MQTT5_ROLE_CREDENTIAL_SECRET_ACCESS_KEY") @@ -399,7 +372,10 @@ def test_mqtt5_ws_cred_environment(self): os.environ["AWS_SECRET_ACCESS_KEY"] = input_secret_access_key os.environ["AWS_SESSION_TOKEN"] = input_session_token # This should load the environment variables we just set - credentials = auth.AwsCredentialsProvider.new_environment() + if use_default_chain: + credentials = auth.AwsCredentialsProvider.new_default_chain() + else: + credentials = auth.AwsCredentialsProvider.new_environment() def sign_function(transform_args, **kwargs): signing_config = auth.AwsSigningConfig( From 3fc70f21d6572abf3399d50e6e47454cd78071d1 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 15:00:21 -0700 Subject: [PATCH 23/27] fix static --- test/test_mqtt5_credentials.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_mqtt5_credentials.py b/test/test_mqtt5_credentials.py index b902e2bb..8a5d24b0 100644 --- a/test/test_mqtt5_credentials.py +++ b/test/test_mqtt5_credentials.py @@ -197,7 +197,6 @@ def test_mqtt5_ws_cred_static(self): input_role_secret_access_key, input_role_session_token ) - credentials = auth.AwsCredentialsProvider.new_default_chain() def sign_function(transform_args, **kwargs): signing_config = auth.AwsSigningConfig( From dfcafc44c2dc09c804d0bb4039cb68c86b3cc10f Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 15:08:05 -0700 Subject: [PATCH 24/27] fix mqtt311 --- test/test_mqtt_credentials.py | 47 ++++++++--------------------------- 1 file changed, 11 insertions(+), 36 deletions(-) diff --git a/test/test_mqtt_credentials.py b/test/test_mqtt_credentials.py index 15335e3e..26487b37 100644 --- a/test/test_mqtt_credentials.py +++ b/test/test_mqtt_credentials.py @@ -136,40 +136,6 @@ def sign_function(transform_args, **kwargs): connection.connect().result(TIMEOUT) connection.disconnect().result(TIMEOUT) - def test_mqtt311_ws_cred_default(self): - input_host_name = _get_env_variable("AWS_TEST_MQTT311_IOT_CORE_HOST") - input_region = _get_env_variable("AWS_TEST_MQTT311_IOT_CORE_REGION") - - credentials = auth.AwsCredentialsProvider.new_default_chain() - - def sign_function(transform_args, **kwargs): - signing_config = auth.AwsSigningConfig( - algorithm=auth.AwsSigningAlgorithm.V4, - signature_type=auth.AwsSignatureType.HTTP_REQUEST_QUERY_PARAMS, - credentials_provider=credentials, - region=input_region, - service="iotdevicegateway", - omit_session_token=True - ) - signing_future = auth.aws_sign_request( - http_request=transform_args.http_request, - signing_config=signing_config) - signing_future.add_done_callback(lambda x: transform_args.set_done(x.exception())) - - elg = EventLoopGroup() - resolver = DefaultHostResolver(elg) - bootstrap = ClientBootstrap(elg, resolver) - client = Client(bootstrap, ClientTlsContext(TlsContextOptions())) - connection = Connection( - client=client, - client_id=create_client_id(), - host_name=input_host_name, - port=int(443), - use_websockets=True, - websocket_handshake_transform=sign_function) - connection.connect().result(TIMEOUT) - connection.disconnect().result(TIMEOUT) - def test_mqtt311_ws_cred_cognito(self): input_cognito_endpoint = _get_env_variable("AWS_TEST_MQTT311_COGNITO_ENDPOINT") input_cognito_identity = _get_env_variable("AWS_TEST_MQTT311_COGNITO_IDENTITY") @@ -300,6 +266,12 @@ def sign_function(transform_args, **kwargs): connection.disconnect().result(TIMEOUT) def test_mqtt311_ws_cred_environment(self): + self._test_mqtt311_ws_cred_environment(use_default_chain = False); + + def test_mqtt311_ws_cred_default(self): + self._test_mqtt311_ws_cred_environment(use_default_chain = True); + + def _test_mqtt311_ws_cred_environment(self, use_default_chain): input_access_key = _get_env_variable("AWS_TEST_MQTT311_ROLE_CREDENTIAL_ACCESS_KEY") input_secret_access_key = _get_env_variable("AWS_TEST_MQTT311_ROLE_CREDENTIAL_SECRET_ACCESS_KEY") input_session_token = _get_env_variable("AWS_TEST_MQTT311_ROLE_CREDENTIAL_SESSION_TOKEN") @@ -314,8 +286,11 @@ def test_mqtt311_ws_cred_environment(self): os.environ["AWS_ACCESS_KEY_ID"] = input_access_key os.environ["AWS_SECRET_ACCESS_KEY"] = input_secret_access_key os.environ["AWS_SESSION_TOKEN"] = input_session_token - # This should load the environment variables we just set - credentials = auth.AwsCredentialsProvider.new_environment() + if use_default_chain: + credentials = auth.AwsCredentialsProvider.new_default_chain() + else: + # This should load the environment variables we just set + credentials = auth.AwsCredentialsProvider.new_environment() signing_config = auth.AwsSigningConfig( algorithm=auth.AwsSigningAlgorithm.V4, signature_type=auth.AwsSignatureType.HTTP_REQUEST_QUERY_PARAMS, From c517439a128878d5c51b7883eff4eb855573f3a6 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 15:40:03 -0700 Subject: [PATCH 25/27] fix something --- test/test_mqtt_credentials.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_mqtt_credentials.py b/test/test_mqtt_credentials.py index 26487b37..e1bd2c68 100644 --- a/test/test_mqtt_credentials.py +++ b/test/test_mqtt_credentials.py @@ -266,10 +266,10 @@ def sign_function(transform_args, **kwargs): connection.disconnect().result(TIMEOUT) def test_mqtt311_ws_cred_environment(self): - self._test_mqtt311_ws_cred_environment(use_default_chain = False); + self._test_mqtt311_ws_cred_environment(use_default_chain=False) def test_mqtt311_ws_cred_default(self): - self._test_mqtt311_ws_cred_environment(use_default_chain = True); + self._test_mqtt311_ws_cred_environment(use_default_chain=True) def _test_mqtt311_ws_cred_environment(self, use_default_chain): input_access_key = _get_env_variable("AWS_TEST_MQTT311_ROLE_CREDENTIAL_ACCESS_KEY") From 267a4b97cd6e09c308c1cf0b411311703ebc0323 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 15:44:16 -0700 Subject: [PATCH 26/27] remove unneeded comment --- .github/workflows/ci.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb7aec11..ee01f1aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,7 +32,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -63,7 +62,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -97,7 +95,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -123,7 +120,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -151,13 +147,11 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ env.CRT_CI_ROLE }} aws-region: ${{ env.AWS_DEFAULT_REGION }} - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: Build ${{ env.PACKAGE_NAME }} run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh @@ -181,7 +175,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -200,7 +193,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -234,7 +226,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -251,7 +242,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -268,7 +258,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -291,7 +280,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -320,7 +308,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -362,7 +349,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: From 7801abebeefea2aa5351d02a5b65a5f0f603a401 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 16:02:31 -0700 Subject: [PATCH 27/27] builder fix --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee01f1aa..ec7bb0f6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,8 +7,8 @@ on: - 'docs' env: - BUILDER_VERSION: test-creds - BUILDER_SOURCE: channels + BUILDER_VERSION: v0.9.67 + BUILDER_SOURCE: releases BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net PACKAGE_NAME: aws-crt-python LINUX_BASE_IMAGE: ubuntu-18-x64