From 213b0a99d6598992110ce6fb551d95152b46ebcd Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 9 Oct 2023 23:23:21 +0000 Subject: [PATCH] finally, changed to linux to devlop --- source/s3_client.c | 4 ---- source/s3_meta_request.c | 21 ++------------------- test/test_s3.py | 36 ++++++++++++------------------------ 3 files changed, 14 insertions(+), 47 deletions(-) diff --git a/source/s3_client.c b/source/s3_client.c index 027b5c21a..f7a558f5a 100644 --- a/source/s3_client.c +++ b/source/s3_client.c @@ -82,7 +82,6 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) { uint64_t part_size = 0; double throughput_target_gbps = 0; int tls_mode; - fprintf(stderr, "###### why???? 1 \n"); if (!PyArg_ParseTuple( args, "OOOOOs#iKdO", @@ -100,7 +99,6 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) { return NULL; } - fprintf(stderr, "###### why???? 2 \n"); struct aws_client_bootstrap *bootstrap = aws_py_get_client_bootstrap(bootstrap_py); if (!bootstrap) { return NULL; @@ -108,13 +106,11 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) { struct aws_credentials_provider *credential_provider = NULL; if (credential_provider_py != Py_None) { - fprintf(stderr, "###### why???? 3 \n"); credential_provider = aws_py_get_credentials_provider(credential_provider_py); if (!credential_provider) { return NULL; } } - fprintf(stderr, "###### why???? 4 \n"); struct aws_signing_config_aws *signing_config = NULL; if (signing_config_py != Py_None) { signing_config = aws_py_get_signing_config(signing_config_py); diff --git a/source/s3_meta_request.c b/source/s3_meta_request.c index 4ccb48064..947d869cd 100644 --- a/source/s3_meta_request.c +++ b/source/s3_meta_request.c @@ -146,10 +146,6 @@ static int s_s3_request_on_body( (void)meta_request; struct s3_meta_request_binding *request_binding = user_data; - bool report_progress; - if (s_record_progress(request_binding, (uint64_t)body->len, &report_progress)) { - return AWS_OP_ERR; - } if (request_binding->recv_file) { /* The callback will be invoked with the right order, so we don't need to seek first. */ if (fwrite((void *)body->ptr, body->len, 1, request_binding->recv_file) < 1) { @@ -163,9 +159,7 @@ static int s_s3_request_on_body( aws_error_name(aws_last_error())); return AWS_OP_ERR; } - if (!report_progress) { - return AWS_OP_SUCCESS; - } + return AWS_OP_SUCCESS; } bool error = true; /*************** GIL ACQUIRE ***************/ @@ -189,17 +183,6 @@ static int s_s3_request_on_body( } Py_DECREF(result); } - if (report_progress) { - /* Hold the GIL before enterring here */ - result = - PyObject_CallMethod(request_binding->py_core, "_on_progress", "(K)", request_binding->size_transferred); - if (!result) { - PyErr_WriteUnraisable(request_binding->py_core); - } else { - Py_DECREF(result); - } - request_binding->size_transferred = 0; - } error = false; done: PyGILState_Release(state); @@ -343,7 +326,7 @@ static void s_s3_request_on_progress( struct s3_meta_request_binding *request_binding = user_data; - bool report_progress; + bool report_progress = false; s_record_progress(request_binding, progress->bytes_transferred, &report_progress); if (report_progress) { diff --git a/test/test_s3.py b/test/test_s3.py index f1a2dee7a..736a1ecf3 100644 --- a/test/test_s3.py +++ b/test/test_s3.py @@ -231,14 +231,14 @@ def test_get_object(self): self._test_s3_put_get_object(request, S3RequestType.GET_OBJECT) def test_put_object(self): - put_body_stream = open(self.temp_put_obj_file_path, "r+b") + put_body_stream = open(self.temp_put_obj_file_path, "rb") content_length = os.stat(self.temp_put_obj_file_path).st_size request = self._put_object_request(put_body_stream, content_length) self._test_s3_put_get_object(request, S3RequestType.PUT_OBJECT) put_body_stream.close() def test_put_object_unknown_content_length(self): - put_body_stream = open(self.temp_put_obj_file_path, "r+b") + put_body_stream = open(self.temp_put_obj_file_path, "rb") content_length = os.stat(self.temp_put_obj_file_path).st_size request = self._put_object_request(put_body_stream, content_length, unknown_content_length=True) self._test_s3_put_get_object(request, S3RequestType.PUT_OBJECT) @@ -257,9 +257,8 @@ def test_put_object_multiple_times(self): for i in range(3): tempfile = self.files.create_file_with_size("temp_file_{}".format(str(i)), 10 * MB) path = "/put_object_test_py_10MB_{}.txt".format(str(i)) - put_body_stream = open(path, "r+b") - content_length = os.stat(path).st_size - request = self._put_object_request(put_body_stream, content_length) + content_length = os.stat(tempfile).st_size + request = self._put_object_request(None, content_length, path=path) s3_request = s3_client.make_request( request=request, type=S3RequestType.PUT_OBJECT, @@ -267,7 +266,6 @@ def test_put_object_multiple_times(self): on_headers=self._on_request_headers, on_body=self._on_request_body) finished_futures.append(s3_request.finished_future) - put_body_stream.close() # request keeps connection alive. delete pointer so connection can shut down del s3_request try: @@ -320,26 +318,20 @@ def test_get_object_filepath(self): os.remove(file.name) def test_put_object_filepath(self): - put_body_stream = open(self.temp_put_obj_file_path, "r+b") content_length = os.stat(self.temp_put_obj_file_path).st_size - request = self._put_object_request(put_body_stream, content_length) + request = self._put_object_request(None, content_length) self._test_s3_put_get_object(request, S3RequestType.PUT_OBJECT, send_filepath=self.temp_put_obj_file_path) - put_body_stream.close() def test_put_object_filepath_unknown_content_length(self): - put_body_stream = open(self.temp_put_obj_file_path, "r+b") content_length = os.stat(self.temp_put_obj_file_path).st_size - request = self._put_object_request(put_body_stream, content_length, unknown_content_length=True) + request = self._put_object_request(None, content_length, unknown_content_length=True) self._test_s3_put_get_object(request, S3RequestType.PUT_OBJECT, send_filepath=self.temp_put_obj_file_path) - put_body_stream.close() def test_put_object_filepath_move(self): # remove the input file when request done tempfile = self.files.create_file_with_size("temp_file", 10 * MB) - put_body_stream = open(tempfile, "r+b") content_length = os.stat(tempfile).st_size - request = self._put_object_request(put_body_stream, content_length) - put_body_stream.close() + request = self._put_object_request(None, content_length) s3_client = s3_client_new(False, self.region, 5 * MB) request_type = S3RequestType.PUT_OBJECT done_future = Future() @@ -359,7 +351,7 @@ def on_done_remove_file(**kwargs): # check result self.assertEqual( - self.data_len, + content_length, self.transferred_len, "the transferred length reported does not match body we sent") self._validate_successful_response(request_type is S3RequestType.PUT_OBJECT) @@ -472,10 +464,8 @@ def test_special_filepath_upload(self): with open(self.special_path, 'wb') as file: file.write(b"a" * 10 * MB) - put_body_stream = open(self.special_path, "r+b") content_length = os.stat(self.special_path).st_size - request = self._put_object_request(put_body_stream, content_length) - put_body_stream.close() + request = self._put_object_request(None, content_length) s3_client = s3_client_new(False, self.region, 5 * MB) request_type = S3RequestType.PUT_OBJECT @@ -508,7 +498,7 @@ def test_special_filepath_upload(self): # check result self.assertEqual( - self.data_len, + content_length, self.transferred_len, "the transferred length reported does not match body we sent") self._validate_successful_response(request_type is S3RequestType.PUT_OBJECT) @@ -519,10 +509,8 @@ def test_non_ascii_filepath_upload(self): with open(self.non_ascii_file_name, 'wb') as file: file.write(b"a" * 10 * MB) - put_body_stream = open(self.non_ascii_file_name, "r+b") content_length = os.stat(self.non_ascii_file_name).st_size - request = self._put_object_request(put_body_stream, content_length) - put_body_stream.close() + request = self._put_object_request(None, content_length) s3_client = s3_client_new(False, self.region, 5 * MB) request_type = S3RequestType.PUT_OBJECT @@ -537,7 +525,7 @@ def test_non_ascii_filepath_upload(self): # check result self.assertEqual( - self.data_len, + content_length, self.transferred_len, "the transferred length reported does not match body we sent") self._validate_successful_response(request_type is S3RequestType.PUT_OBJECT)