Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

5.17 Backports #8383

Merged
merged 9 commits into from
Sep 19, 2024
Merged

5.17 Backports #8383

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
fd3bda1
NC | NSFS | Versioning | Fix Bug | Return 405 for get/head specific d…
shirady Sep 10, 2024
c9a5483
CI | NSFS | Ceph S3 Tests
shirady Sep 15, 2024
99a0b6d
NSFS | fix copy_object issues
nadavMiz Sep 12, 2024
fd7a6ba
Delete object version id concurrency
romayalon Sep 16, 2024
01a7c43
CI | NSFS | Ceph S3 Tests | Move tests from pending list to blacklist
shirady Sep 17, 2024
cbcf5fd
NSFS | remove prev external attribute
nadavMiz Sep 11, 2024
9d3f856
NC | NSFS | Versioning | Fix Bug 8333 | Delete object with version id
shirady Sep 11, 2024
e01a57d
NC | NSFS | Versioning | Delete Objects | Add a Test | GPFS
shirady Sep 17, 2024
4808aac
NSFS | GPFS | fix delete latest object twice on GPFS
nadavMiz Sep 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/endpoint/s3/s3_errors.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -609,6 +609,7 @@ S3Error.RPC_ERRORS_TO_S3 = Object.freeze({
NO_SUCH_TAG: S3Error.NoSuchTagSet,
INVALID_ENCODING_TYPE: S3Error.InvalidEncodingType,
INVALID_TARGET_BUCKET: S3Error.InvalidTargetBucketForLogging,
METHOD_NOT_ALLOWED: S3Error.MethodNotAllowed,
});

exports.S3Error = S3Error;
46 changes: 20 additions & 26 deletions src/endpoint/s3/s3_rest.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -383,7 +383,7 @@ function parse_op_name(req) {
return `${method}_object`;
}

function handle_error(req, res, err) {
function _prepare_error(req, res, err) {
let s3err =
((err instanceof S3Error) && err) ||
new S3Error(S3Error.RPC_ERRORS_TO_S3[err.rpc_code] || S3Error.InternalError);
Expand All @@ -393,19 +393,26 @@ function handle_error(req, res, err) {
s3err.detail = err.rpc_data.detail;
}

if (s3err.rpc_data) {
if (s3err.rpc_data.etag) {
if (err.rpc_data) {
if (err.rpc_data.etag) {
if (res.headersSent) {
dbg.log0('Sent reply in body, bit too late for Etag header');
} else {
res.setHeader('ETag', s3err.rpc_data.etag);
res.setHeader('ETag', err.rpc_data.etag);
}
}
if (s3err.rpc_data.last_modified) {
if (err.rpc_data.last_modified) {
if (res.headersSent) {
dbg.log0('Sent reply in body, bit too late for Last-Modified header');
} else {
res.setHeader('Last-Modified', time_utils.format_http_header_date(new Date(s3err.rpc_data.last_modified)));
res.setHeader('Last-Modified', time_utils.format_http_header_date(new Date(err.rpc_data.last_modified)));
}
}
if (err.rpc_data.delete_marker) {
if (res.headersSent) {
dbg.log0('Sent reply in body, bit too late for x-amz-delete-marker header');
} else {
res.setHeader('x-amz-delete-marker', String(err.rpc_data.delete_marker));
}
}
}
Expand All @@ -418,6 +425,12 @@ function handle_error(req, res, err) {
usage_report.s3_errors_info.total_errors += 1;
usage_report.s3_errors_info[s3err.code] = (usage_report.s3_errors_info[s3err.code] || 0) + 1;

return s3err;
}

function handle_error(req, res, err) {
const s3err = _prepare_error(req, res, err);

const reply = s3err.reply(req.originalUrl, req.request_id);
dbg.error('S3 ERROR', reply,
req.method, req.originalUrl,
Expand All @@ -436,26 +449,7 @@ function handle_error(req, res, err) {
}

async function _handle_html_response(req, res, err) {
let s3err =
((err instanceof S3Error) && err) ||
new S3Error(S3Error.RPC_ERRORS_TO_S3[err.rpc_code] || S3Error.InternalError);

if (s3err.rpc_data) {
if (s3err.rpc_data.etag) {
res.setHeader('ETag', s3err.rpc_data.etag);
}
if (s3err.rpc_data.last_modified) {
res.setHeader('Last-Modified', time_utils.format_http_header_date(new Date(s3err.rpc_data.last_modified)));
}
}

// md_conditions used for PUT/POST/DELETE should return PreconditionFailed instead of NotModified
if (s3err.code === 'NotModified' && req.method !== 'HEAD' && req.method !== 'GET') {
s3err = new S3Error(S3Error.PreconditionFailed);
}

usage_report.s3_errors_info.total_errors += 1;
usage_report.s3_errors_info[s3err.code] = (usage_report.s3_errors_info[s3err.code] || 0) + 1;
const s3err = _prepare_error(req, res, err);

const reply = `<html> \
<head><title>${s3err.http_code} ${s3err.code}</title></head> \
Expand Down
125 changes: 48 additions & 77 deletions src/sdk/namespace_fs.js
View file
Open in desktop

Large diffs are not rendered by default.

33 changes: 18 additions & 15 deletions src/sdk/object_sdk.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,8 +106,8 @@ class ObjectSDK {
* in order to handle aborting requests gracefully. The `abort_controller` member will
* be used to signal async flows that abort was detected.
* @see {@link https://nodejs.org/docs/latest/api/globals.html#class-abortcontroller}
* @param {import('http').IncomingMessage} req
* @param {import('http').ServerResponse} res
* @param {import('http').IncomingMessage} req
* @param {import('http').ServerResponse} res
*/
setup_abort_controller(req, res) {
res.once('error', err => {
Expand Down Expand Up @@ -158,7 +158,7 @@ class ObjectSDK {
}

/**
* @param {string} name
* @param {string} name
* @returns {Promise<nb.Namespace>}
*/
async _get_bucket_namespace(name) {
Expand Down Expand Up @@ -268,7 +268,7 @@ class ObjectSDK {
return Boolean(fs_root_path || fs_root_path === '');
}

// validates requests for non nsfs buckets from accounts which are nsfs_only
// validates requests for non nsfs buckets from accounts which are nsfs_only
has_non_nsfs_bucket_access(account, ns) {
dbg.log1('validate_non_nsfs_bucket: ', account, ns?.write_resource?.resource);
if (!account) return false;
Expand Down Expand Up @@ -524,7 +524,7 @@ class ObjectSDK {
/**
* Calls the op and report time and error to stats collector.
* on_success can be added to update read/write stats (but on_success shouln't throw)
*
*
* @template T
* @param {{
* op_name: string;
Expand Down Expand Up @@ -642,7 +642,9 @@ class ObjectSDK {
params.content_type = source_md.content_type;
}
try {
if (params.xattr) params.xattr = _.omitBy(params.xattr, (val, name) => name.startsWith('noobaa-namespace'));
//omitBy iterates all xattr calling startsWith on them. this can include symbols such as XATTR_SORT_SYMBOL.
//in that case startsWith will not apply
if (params.xattr) params.xattr = _.omitBy(params.xattr, (val, name) => name.startsWith?.('noobaa-namespace'));
} catch (e) {
dbg.log3("Got an error while trying to omitBy param.xattr:", params.xattr, "error:", e);
}
Expand All @@ -658,19 +660,14 @@ class ObjectSDK {
params.copy_source.bucket = actual_source_ns.get_bucket(bucket);
params.copy_source.obj_id = source_md.obj_id;
params.copy_source.version_id = source_md.version_id;
if (source_ns instanceof NamespaceFS) {
params.copy_source.nsfs_copy_fallback = () => {
this._populate_nsfs_copy_fallback({ source_params, source_ns, params });
params.copy_source = null;
};
}
} else {
// source cannot be copied directly (different plaforms, accounts, etc.)
// set the source_stream to read from the copy source
// Source params need these for read operations
source_params.object_md = source_md;
source_params.obj_id = source_md.obj_id;
source_params.version_id = source_md.version_id;
source_params.bucket = actual_source_ns.get_bucket(bucket);
// param size is needed when doing an upload. Can be overrided during ranged writes
params.size = source_md.size;

Expand All @@ -684,7 +681,13 @@ class ObjectSDK {

// if the source namespace is NSFS then we need to pass the read_object_stream the read_stream
if (source_ns instanceof NamespaceFS) {
this._populate_nsfs_copy_fallback({ source_params, source_ns, params });
if (target_ns instanceof NamespaceFS) {
params.source_ns = actual_source_ns;
params.source_params = source_params;
} else {
//this._populate_nsfs_copy_fallback({ source_params, source_ns, params });
throw new Error('TODO fix _populate_nsfs_copy_fallback');
}
} else {
params.source_stream = await source_ns.read_object_stream(source_params, this);
}
Expand All @@ -701,9 +704,9 @@ class ObjectSDK {
}
}

// nsfs copy_object & server side copy consisted of link and a fallback to
// nsfs copy_object & server side copy consisted of link and a fallback to
// read stream and then upload stream
// nsfs copy object when can't server side copy - fallback directly
// nsfs copy object when can't server side copy - fallback directly
_populate_nsfs_copy_fallback({ source_ns, params, source_params }) {
const read_stream = new stream.PassThrough();
source_ns.read_object_stream(source_params, this, read_stream)
Expand Down
59 changes: 58 additions & 1 deletion src/test/system_tests/ceph_s3_tests/s3-tests-lists/nsfs_s3_tests_black_list.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -349,4 +349,61 @@ s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_resourc
s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_wrong_resource_tag_deny
s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_resource_tag_princ_tag
s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_resource_tag_copy_obj
s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_role_resource_tag
s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_role_resource_tag
s3tests_boto3/functional/test_s3.py::test_post_object_invalid_signature
s3tests_boto3/functional/test_s3.py::test_post_object_invalid_access_key
s3tests_boto3/functional/test_s3.py::test_post_object_missing_policy_condition
s3tests_boto3/functional/test_s3.py::test_post_object_request_missing_policy_specified_field
s3tests_boto3/functional/test_s3.py::test_post_object_expired_policy
s3tests_boto3/functional/test_s3.py::test_post_object_invalid_request_field_value
s3tests_boto3/functional/test_s3.py::test_post_object_authenticated_request_bad_access_key
s3tests_boto3/functional/test_s3.py::test_sse_s3_default_post_object_authenticated_request
s3tests_boto3/functional/test_s3.py::test_sse_kms_default_post_object_authenticated_request
s3tests_boto3/functional/test_s3.py::test_post_object_upload_size_rgw_chunk_size_bug
s3tests_boto3/functional/test_s3.py::test_post_object_wrong_bucket
s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration
s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_versioning_enabled
s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_tags2
s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_versioned_tags2
s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_noncur_tags1
s3tests_boto3/functional/test_s3.py::test_lifecycle_set_date
s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_header_put
s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_header_head
s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_header_tags_head
s3tests_boto3/functional/test_s3.py::test_lifecycle_transition_set_invalid_date
s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_newer_noncurrent
s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_size_gt
s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_size_lt
s3tests_boto3/functional/test_s3.py::test_object_lock_put_obj_lock_invalid_mode
s3tests_boto3/functional/test_s3.py::test_object_lock_get_obj_retention_iso8601
s3tests_boto3/functional/test_s3.py::test_object_lock_delete_object_with_retention_and_marker
s3tests_boto3/functional/test_s3.py::test_object_lock_multi_delete_object_with_retention
s3tests_boto3/functional/test_s3.py::test_object_lock_put_legal_hold
s3tests_boto3/functional/test_s3.py::test_object_lock_get_legal_hold
s3tests_boto3/functional/test_s3.py::test_object_lock_changing_mode_from_governance_with_bypass
s3tests_boto3/functional/test_s3.py::test_object_lock_changing_mode_from_governance_without_bypass
s3tests_boto3/functional/test_s3.py::test_object_lock_changing_mode_from_compliance
s3tests_boto3/functional/test_s3.py::test_object_lock_delete_multipart_object_with_retention
s3tests_boto3/functional/test_s3.py::test_object_lock_delete_multipart_object_with_legal_hold_on
s3tests_boto3/functional/test_s3.py::test_sse_kms_method_head
s3tests_boto3/functional/test_s3.py::test_sse_s3_default_upload_1b
s3tests_boto3/functional/test_s3.py::test_sse_s3_default_upload_1kb
s3tests_boto3/functional/test_s3.py::test_sse_s3_default_upload_1mb
s3tests_boto3/functional/test_s3.py::test_sse_s3_default_upload_8mb
s3tests_boto3/functional/test_s3.py::test_sse_kms_default_upload_1b
s3tests_boto3/functional/test_s3.py::test_sse_kms_default_upload_1kb
s3tests_boto3/functional/test_s3.py::test_sse_kms_default_upload_1mb
s3tests_boto3/functional/test_s3.py::test_sse_kms_default_upload_8mb
s3tests_boto3/functional/test_s3.py::test_sse_s3_default_method_head
s3tests_boto3/functional/test_s3.py::test_sse_s3_default_multipart_upload
s3tests_boto3/functional/test_s3.py::test_sse_s3_encrypted_upload_1b
s3tests_boto3/functional/test_s3.py::test_sse_s3_encrypted_upload_1kb
s3tests_boto3/functional/test_s3.py::test_sse_s3_encrypted_upload_1mb
s3tests_boto3/functional/test_s3.py::test_sse_s3_encrypted_upload_8mb
s3tests_boto3/functional/test_s3.py::test_cors_presigned_get_object
s3tests_boto3/functional/test_s3.py::test_cors_presigned_get_object_tenant
s3tests_boto3/functional/test_s3.py::test_cors_presigned_put_object
s3tests_boto3/functional/test_s3.py::test_cors_presigned_put_object_with_acl
s3tests_boto3/functional/test_s3.py::test_cors_presigned_put_object_tenant
s3tests_boto3/functional/test_s3.py::test_cors_presigned_put_object_tenant_with_acl

Loading