Skip to content

Commit

Permalink
finally, changed to linux to devlop
Browse files Browse the repository at this point in the history
  • Loading branch information
@TingDaoK
TingDaoK committed Oct 9, 2023
1 parent 6b90254 commit 213b0a9
Show file tree
Hide file tree
Showing 3 changed files with 14 additions and 47 deletions.
4 changes: 0 additions & 4 deletions source/s3_client.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,6 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) {
uint64_t part_size = 0;
double throughput_target_gbps = 0;
int tls_mode;
fprintf(stderr, "###### why???? 1 \n");
if (!PyArg_ParseTuple(
args,
"OOOOOs#iKdO",
Expand All @@ -100,21 +99,18 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) {
return NULL;
}

fprintf(stderr, "###### why???? 2 \n");
struct aws_client_bootstrap *bootstrap = aws_py_get_client_bootstrap(bootstrap_py);
if (!bootstrap) {
return NULL;
}

struct aws_credentials_provider *credential_provider = NULL;
if (credential_provider_py != Py_None) {
fprintf(stderr, "###### why???? 3 \n");
credential_provider = aws_py_get_credentials_provider(credential_provider_py);
if (!credential_provider) {
return NULL;
}
}
fprintf(stderr, "###### why???? 4 \n");
struct aws_signing_config_aws *signing_config = NULL;
if (signing_config_py != Py_None) {
signing_config = aws_py_get_signing_config(signing_config_py);
Expand Down
21 changes: 2 additions & 19 deletions source/s3_meta_request.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,10 +146,6 @@ static int s_s3_request_on_body(
(void)meta_request;
struct s3_meta_request_binding *request_binding = user_data;

bool report_progress;
if (s_record_progress(request_binding, (uint64_t)body->len, &report_progress)) {
return AWS_OP_ERR;
}
if (request_binding->recv_file) {
/* The callback will be invoked with the right order, so we don't need to seek first. */
if (fwrite((void *)body->ptr, body->len, 1, request_binding->recv_file) < 1) {
Expand All @@ -163,9 +159,7 @@ static int s_s3_request_on_body(
aws_error_name(aws_last_error()));
return AWS_OP_ERR;
}
if (!report_progress) {
return AWS_OP_SUCCESS;
}
return AWS_OP_SUCCESS;
}
bool error = true;
/*************** GIL ACQUIRE ***************/
Expand All @@ -189,17 +183,6 @@ static int s_s3_request_on_body(
}
Py_DECREF(result);
}
if (report_progress) {
/* Hold the GIL before enterring here */
result =
PyObject_CallMethod(request_binding->py_core, "_on_progress", "(K)", request_binding->size_transferred);
if (!result) {
PyErr_WriteUnraisable(request_binding->py_core);
} else {
Py_DECREF(result);
}
request_binding->size_transferred = 0;
}
error = false;
done:
PyGILState_Release(state);
Expand Down Expand Up @@ -343,7 +326,7 @@ static void s_s3_request_on_progress(

struct s3_meta_request_binding *request_binding = user_data;

bool report_progress;
bool report_progress = false;
s_record_progress(request_binding, progress->bytes_transferred, &report_progress);

if (report_progress) {
Expand Down
36 changes: 12 additions & 24 deletions test/test_s3.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,14 +231,14 @@ def test_get_object(self):
self._test_s3_put_get_object(request, S3RequestType.GET_OBJECT)

def test_put_object(self):
put_body_stream = open(self.temp_put_obj_file_path, "r+b")
put_body_stream = open(self.temp_put_obj_file_path, "rb")
content_length = os.stat(self.temp_put_obj_file_path).st_size
request = self._put_object_request(put_body_stream, content_length)
self._test_s3_put_get_object(request, S3RequestType.PUT_OBJECT)
put_body_stream.close()

def test_put_object_unknown_content_length(self):
put_body_stream = open(self.temp_put_obj_file_path, "r+b")
put_body_stream = open(self.temp_put_obj_file_path, "rb")
content_length = os.stat(self.temp_put_obj_file_path).st_size
request = self._put_object_request(put_body_stream, content_length, unknown_content_length=True)
self._test_s3_put_get_object(request, S3RequestType.PUT_OBJECT)
Expand All @@ -257,17 +257,15 @@ def test_put_object_multiple_times(self):
for i in range(3):
tempfile = self.files.create_file_with_size("temp_file_{}".format(str(i)), 10 * MB)
path = "/put_object_test_py_10MB_{}.txt".format(str(i))
put_body_stream = open(path, "r+b")
content_length = os.stat(path).st_size
request = self._put_object_request(put_body_stream, content_length)
content_length = os.stat(tempfile).st_size
request = self._put_object_request(None, content_length, path=path)
s3_request = s3_client.make_request(
request=request,
type=S3RequestType.PUT_OBJECT,
send_filepath=tempfile,
on_headers=self._on_request_headers,
on_body=self._on_request_body)
finished_futures.append(s3_request.finished_future)
put_body_stream.close()
# request keeps connection alive. delete pointer so connection can shut down
del s3_request
try:
Expand Down Expand Up @@ -320,26 +318,20 @@ def test_get_object_filepath(self):
os.remove(file.name)

def test_put_object_filepath(self):
put_body_stream = open(self.temp_put_obj_file_path, "r+b")
content_length = os.stat(self.temp_put_obj_file_path).st_size
request = self._put_object_request(put_body_stream, content_length)
request = self._put_object_request(None, content_length)
self._test_s3_put_get_object(request, S3RequestType.PUT_OBJECT, send_filepath=self.temp_put_obj_file_path)
put_body_stream.close()

def test_put_object_filepath_unknown_content_length(self):
put_body_stream = open(self.temp_put_obj_file_path, "r+b")
content_length = os.stat(self.temp_put_obj_file_path).st_size
request = self._put_object_request(put_body_stream, content_length, unknown_content_length=True)
request = self._put_object_request(None, content_length, unknown_content_length=True)
self._test_s3_put_get_object(request, S3RequestType.PUT_OBJECT, send_filepath=self.temp_put_obj_file_path)
put_body_stream.close()

def test_put_object_filepath_move(self):
# remove the input file when request done
tempfile = self.files.create_file_with_size("temp_file", 10 * MB)
put_body_stream = open(tempfile, "r+b")
content_length = os.stat(tempfile).st_size
request = self._put_object_request(put_body_stream, content_length)
put_body_stream.close()
request = self._put_object_request(None, content_length)
s3_client = s3_client_new(False, self.region, 5 * MB)
request_type = S3RequestType.PUT_OBJECT
done_future = Future()
Expand All @@ -359,7 +351,7 @@ def on_done_remove_file(**kwargs):

# check result
self.assertEqual(
self.data_len,
content_length,
self.transferred_len,
"the transferred length reported does not match body we sent")
self._validate_successful_response(request_type is S3RequestType.PUT_OBJECT)
Expand Down Expand Up @@ -472,10 +464,8 @@ def test_special_filepath_upload(self):
with open(self.special_path, 'wb') as file:
file.write(b"a" * 10 * MB)

put_body_stream = open(self.special_path, "r+b")
content_length = os.stat(self.special_path).st_size
request = self._put_object_request(put_body_stream, content_length)
put_body_stream.close()
request = self._put_object_request(None, content_length)
s3_client = s3_client_new(False, self.region, 5 * MB)
request_type = S3RequestType.PUT_OBJECT

Expand Down Expand Up @@ -508,7 +498,7 @@ def test_special_filepath_upload(self):

# check result
self.assertEqual(
self.data_len,
content_length,
self.transferred_len,
"the transferred length reported does not match body we sent")
self._validate_successful_response(request_type is S3RequestType.PUT_OBJECT)
Expand All @@ -519,10 +509,8 @@ def test_non_ascii_filepath_upload(self):
with open(self.non_ascii_file_name, 'wb') as file:
file.write(b"a" * 10 * MB)

put_body_stream = open(self.non_ascii_file_name, "r+b")
content_length = os.stat(self.non_ascii_file_name).st_size
request = self._put_object_request(put_body_stream, content_length)
put_body_stream.close()
request = self._put_object_request(None, content_length)
s3_client = s3_client_new(False, self.region, 5 * MB)
request_type = S3RequestType.PUT_OBJECT

Expand All @@ -537,7 +525,7 @@ def test_non_ascii_filepath_upload(self):

# check result
self.assertEqual(
self.data_len,
content_length,
self.transferred_len,
"the transferred length reported does not match body we sent")
self._validate_successful_response(request_type is S3RequestType.PUT_OBJECT)
Expand Down

0 comments on commit 213b0a9

Please sign in to comment.