CLDSRV-632 ✨ put metadata edge cases

[DarkIsDude]

Description

Reactivate some tests that were disable because some edge case were not covered.

Motivation and context

Manage 2 use case under the put metadata route:

• Manage null version when an object is created before bucket versioning and put metadata called after bucket versioning enabled

/* eslint-disable no-console */
const util = require('util');

const { makeBackbeatRequest: makeBackbeatRequestBase } = require('./tests/functional/raw-node/utils/makeRequest');
const { models } = require('arsenal');
const { ObjectMD } = models;
const BucketUtility = require('./tests/functional/aws-node-sdk/lib/utility/bucket-util');

const makeBackbeatRequest = util.promisify(makeBackbeatRequestBase);

function objectMDFromRequestBody(data) {
    const bodyStr = JSON.parse(data.body).Body;
    return new ObjectMD(JSON.parse(bodyStr));
}

async function main() {
    const OBJECT_KEY = 'test-object';
    const BUCKET_NAME = 'test-bucket';
    const TEST_DATA = 'This is a test object for replication';

    const authCredentials = {
        accessKey: 'BJ8Q0L35PRJ92ABA2K0B',
        secretKey: 'kTgcfEaLjxvrLN5EKVcTnb4Ac046FU1m=33/baf1',
    };


    const bucketUtil = new BucketUtility('local-test-vault-s3', { signatureVersion: 'v4' });
    const s3 = bucketUtil.s3;

    console.info('Starting test for object metadata replication...');
    console.info('Checking if bucket exists...');
    await bucketUtil.emptyIfExists(BUCKET_NAME);

    if (await bucketUtil.bucketExists(BUCKET_NAME)) {
        console.info('Deleting bucket...');
        await s3.deleteBucket({ Bucket: BUCKET_NAME }).promise();
    }

    console.info('Creating bucket...');
    await s3.createBucket({ Bucket: BUCKET_NAME }).promise();
 
    console.info('Putting object with versioning disabled...');
    await s3.putObject({ Bucket: BUCKET_NAME, Key: OBJECT_KEY, Body: Buffer.from(TEST_DATA) }).promise();

    console.info('Enabling versioning on bucket...');
    await s3.putBucketVersioning({ Bucket: BUCKET_NAME, VersioningConfiguration: { Status: 'Enabled' } }).promise();
    const versionId = null;
    console.info('Retrieve metadata for the object with versioning enabled...');
    const data = await makeBackbeatRequest({
        method: 'GET',
        resourceType: 'metadata',
        bucket: BUCKET_NAME,
        objectKey: OBJECT_KEY,
        queryObj: {
            versionId,
        },
        authCredentials,
    });

    const objMD = objectMDFromRequestBody(data)
        .setContentLanguage('fr-FR')
        .getSerialized();

    console.info('Object metadata retrieved successfully:', objMD);
    console.info('Updating object metadata...');

    objMD.tags = {
        'fuck': 'test-value',
    };

    await makeBackbeatRequest({
        method: 'PUT',
        resourceType: 'metadata',
        bucket: BUCKET_NAME,
        objectKey: OBJECT_KEY,
        queryObj: {
            versionId,
        },
        authCredentials,
        requestBody: objMD,
    });

    console.info('Object metadata updated successfully.');

    const versions = await s3.listObjectVersions({
        Bucket: BUCKET_NAME,
        Prefix: OBJECT_KEY,
    }).promise();

    console.info({ versions });
}

main();

• Add missing metadata

/* eslint-disable no-console */
const util = require('util');

const { makeBackbeatRequest: makeBackbeatRequestBase } = require('./tests/functional/raw-node/utils/makeRequest');
const { models } = require('arsenal');
const { ObjectMD } = models;
const BucketUtility = require('./tests/functional/aws-node-sdk/lib/utility/bucket-util');

const makeBackbeatRequest = util.promisify(makeBackbeatRequestBase);

/**
 * The final result should be to have two versions of the object:
 * 1. The original version created when versioning was disabled (null versionId)
 * 2. A new version created when versioning was re-enabled (with a VersionId)
 * The new version should have the updated metadata (ContentLanguage set to 'fr-FR')
 * while the original version should remain unchanged.
 **/

const OBJECT_KEY_ROOT = 'test-object-root';
const OBJECT_KEY = 'test-object';
const BUCKET_NAME = 'test-bucket';
const TEST_DATA = 'This is a test object for replication';

const authCredentials = {
    accessKey: 'BJ8Q0L35PRJ92ABA2K0B',
    secretKey: 'kTgcfEaLjxvrLN5EKVcTnb4Ac046FU1m=33/baf1',
};

function objectMDFromRequestBody(data) {
    const bodyStr = JSON.parse(data.body).Body;
    return new ObjectMD(JSON.parse(bodyStr));
}

async function main() {
    const bucketUtil = new BucketUtility('local-test-vault-s3', { signatureVersion: 'v4' });
    const s3 = bucketUtil.s3;

    // await initTest(s3, bucketUtil);
    await updateObjectMetadata(s3);
}

async function initTest(s3, bucketUtil) {
    if (await bucketUtil.bucketExists(BUCKET_NAME)) {
        await bucketUtil.emptyIfExists(BUCKET_NAME);
        await s3.deleteBucket({ Bucket: BUCKET_NAME }).promise();
    }

    await s3.createBucket({ Bucket: BUCKET_NAME }).promise();
    await s3.putBucketVersioning({ Bucket: BUCKET_NAME, VersioningConfiguration: { Status: 'Enabled' } }).promise();
    await s3.putObject({ Bucket: BUCKET_NAME, Key: OBJECT_KEY_ROOT, Body: Buffer.from(TEST_DATA) }).promise();
    await s3.putBucketVersioning({ Bucket: BUCKET_NAME, VersioningConfiguration: { Status: 'Suspended' } }).promise();
    await s3.putObject({ Bucket: BUCKET_NAME, Key: OBJECT_KEY, Body: Buffer.from(TEST_DATA) }).promise();
    await s3.putBucketVersioning({ Bucket: BUCKET_NAME, VersioningConfiguration: { Status: 'Enabled' } }).promise();
}

async function updateObjectMetadata(s3) {
        const data = await makeBackbeatRequest({
        method: 'GET',
        resourceType: 'metadata',
        bucket: BUCKET_NAME,
        objectKey: OBJECT_KEY,
        queryObj: {
            versionId: null,
        },
        authCredentials,
    });

    const objMD = objectMDFromRequestBody(data)
        .setContentLanguage('fr-FR')
        .getSerialized();

    await makeBackbeatRequest({
        method: 'PUT',
        resourceType: 'metadata',
        bucket: BUCKET_NAME,
        objectKey: OBJECT_KEY,
        queryObj: {
            versionId: '393832343336313437373734363139393939393952473030312020333061396264',
        },
        authCredentials,
        requestBody: objMD,
    });

    const versions = await s3.listObjectVersions({
        Bucket: BUCKET_NAME,
        Prefix: OBJECT_KEY,
    }).promise();

    console.info({ Versions: versions.Versions });
 }

async function run() {
    try {
        await main();
    } catch (e) {
        console.error('Error running test:', e);
        process.exit(1);
    }
}

run();

Related issues

https://scality.atlassian.net/browse/CLDSRV-632
scality/Arsenal#2490

[bert-e]

Hello darkisdude,

My role is to assist you with the merge of this
pull request. Please type @bert-e help to get information
on this process, or consult the user documentation.

Available options

name	description	privileged	authored
/after_pull_request	Wait for the given pull request id to be merged before continuing with the current one.
/bypass_author_approval	Bypass the pull request author's approval	⭐
/bypass_build_status	Bypass the build and test status	⭐
/bypass_commit_size	Bypass the check on the size of the changeset TBA	⭐
/bypass_incompatible_branch	Bypass the check on the source branch prefix	⭐
/bypass_jira_check	Bypass the Jira issue check	⭐
/bypass_peer_approval	Bypass the pull request peers' approval	⭐
/bypass_leader_approval	Bypass the pull request leaders' approval	⭐
/approve	Instruct Bert-E that the author has approved the pull request.		✍️
/create_pull_requests	Allow the creation of integration pull requests.
/create_integration_branches	Allow the creation of integration branches.
/no_octopus	Prevent Wall-E from doing any octopus merge and use multiple consecutive merge instead
/unanimity	Change review acceptance criteria from one reviewer at least to all reviewers
/wait	Instruct Bert-E not to run until further notice.

Available commands

name	description	privileged
/help	Print Bert-E's manual in the pull request.
/status	Print Bert-E's current status in the pull request TBA
/clear	Remove all comments from Bert-E from the history TBA
/retry	Re-start a fresh build TBA
/build	Re-start a fresh build TBA
/force_reset	Delete integration branches & pull requests, and restart merge process from the beginning.
/reset	Try to remove integration branches unless there are commits on them which do not appear on the source branch.

Status report is not available.

[bert-e]

Incorrect fix version

The Fix Version/s in issue CLDSRV-632 contains:

• None

Considering where you are trying to merge, I ignored possible hotfix versions and I expected to find:

• 9.0.23

• 9.1.0

Please check the Fix Version/s of CLDSRV-632, or the target
branch of this pull request.

[bert-e]

Incorrect fix version

The Fix Version/s in issue CLDSRV-632 contains:

• None

Considering where you are trying to merge, I ignored possible hotfix versions and I expected to find:

• 9.1.0

Please check the Fix Version/s of CLDSRV-632, or the target
branch of this pull request.

[codecov]

Codecov Report

❌ Patch coverage is 92.00000% with 2 lines in your changes missing coverage. Please review.
✅ Project coverage is 83.74%. Comparing base (a8d5672) to head (4357e13).

Files with missing lines	Patch %	Lines
lib/routes/routeBackbeat.js	91.66%	2 Missing ⚠️

Additional details and impacted files

Files with missing lines	Coverage Δ
lib/api/apiUtils/object/createAndStoreObject.js	95.65% <ø> (ø)
lib/api/apiUtils/object/versioning.js	97.10% <100.00%> (+0.01%)	⬆️
lib/routes/routeBackbeat.js	76.33% <91.66%> (+0.53%)	⬆️

@@                 Coverage Diff                 @@
##           development/9.1    #5913      +/-   ##
===================================================
+ Coverage            83.72%   83.74%   +0.01%     
===================================================
  Files                  191      191              
  Lines                12233    12258      +25     
===================================================
+ Hits                 10242    10265      +23     
- Misses                1991     1993       +2     

Flag	Coverage Δ
ceph-backend-test	64.64% <92.00%> (+0.07%)	⬆️
file-ft-tests	66.86% <36.00%> (-0.07%)	⬇️
kmip-ft-tests	27.17% <20.00%> (-0.02%)	⬇️
mongo-v0-ft-tests	68.19% <36.00%> (-0.05%)	⬇️
mongo-v1-ft-tests	68.18% <36.00%> (-0.07%)	⬇️
multiple-backend	34.40% <92.00%> (+0.15%)	⬆️
sur-tests	34.76% <20.00%> (-0.89%)	⬇️
sur-tests-inflights	36.68% <20.00%> (-0.07%)	⬇️
unit	68.31% <68.00%> (-0.01%)	⬇️
utapi-v2-tests	33.54% <20.00%> (-0.03%)	⬇️

Flags with carried forward coverage won't be shown. Click here to find out more.

🚀 New features to boost your workflow:

• 📦 JS Bundle Analysis: Save yourself from yourself by tracking and limiting bundle sizes in JS merges.

[francoisferrand]

when does this !objMd case happen? Before this function is called, we already call standardMetadataValidateBucketAndObj (line 1655), which should return the object already...
(and looking at versioningPreprocessing, it seems that i knows how to handle objMd=nil as well as the case where object is/isn't the master?)

[DarkIsDude]

@francoisferrand when you do a PUT on PutMD and the object does not exists. This case can happens when it's an empty object

[francoisferrand]

still not clear why we need the conditional, and not (more) systematically call versioningPreprocessing.
in particular, this function is responsible to handle version-suspended case (i.e. delete previous version). Or is this case objMD == nil a way to detect the insertion case (i.e. create a new version) instead of the udpate case (update -internal- metadata of an existing version, e.g. for replication status or transition)?

may help to have a comment explaining what use-case this "branch" handles.

This case can happens when it's an empty object

do you mean "empty" as in "no data"? I don't understand how it would/should affect the behavior here :-/

[DarkIsDude]

@francoisferrand if we have an objMd (so an object already exists), we don't want to call this function as this function will create a new version and we don't want that. So I would say it's more to detect indeed the insertion case and the update case.

Do you think a comment like If we create a new version of an object (so objMd is null), we should make sure that the masterVersion is versionned. If an object already exists, we just want to update the metadata of the existing object and not create a new one ?

The versionning is just an "optimisation" but can be removed 🙏.

[bert-e]

Incorrect fix version

The Fix Version/s in issue CLDSRV-632 contains:

• None

Considering where you are trying to merge, I ignored possible hotfix versions and I expected to find:

• 9.1.1

Please check the Fix Version/s of CLDSRV-632, or the target
branch of this pull request.

[bert-e]

Waiting for approval

The following approvals are needed before I can proceed with the merge:

• the author

• 2 peers

[francoisferrand]

somehow I am not sure this covers all cases - and not confident everything is fully tests: there are tests, but logic is quite intricate with the different versioning cases + different listings/... and all the flags: isNull, isNull2, ...

→ would be worth reviewing the tests to see if we miss some cases (e.g. all the combinations of versioning disabled/enabled/suspended, creating vs updating an object, with previous version/nullVersion/no previous version...). May help to look at putObject tests for inspiration on the test case?
→ since we have multiple backend, and the separation is sometimes not clear (or not clearly documented), we should make sure this test is run both with mongo and metadata backends: is it the case?

[francoisferrand]

the semantics of putObject in metadata layer (either backends, i.e. metadata or mongodbClientInterface) are not clear or precisely defined : does setting this not interract (in some weird or unacceptable way?) the behavior of the versionId option?

[DarkIsDude]

As far as I checked, this seems not be the case. But maybe I'm missing something here 😬. Maybe the second review will have more insight

[francoisferrand]

still not clear why we need the conditional, and not (more) systematically call versioningPreprocessing.
in particular, this function is responsible to handle version-suspended case (i.e. delete previous version). Or is this case objMD == nil a way to detect the insertion case (i.e. create a new version) instead of the udpate case (update -internal- metadata of an existing version, e.g. for replication status or transition)?

may help to have a comment explaining what use-case this "branch" handles.

This case can happens when it's an empty object

do you mean "empty" as in "no data"? I don't understand how it would/should affect the behavior here :-/

[francoisferrand]

in other uses of versioningPreprocessing (in putObject / copyObject / putMpu) we set multiple (and different) fields:

			metadataStoreParams.versionId = options.versionId;
            metadataStoreParams.versioning = options.versioning;
            metadataStoreParams.isNull = options.isNull;
            metadataStoreParams.deleteNullKey = options.deleteNullKey;
            if (options.extraMD) {
                Object.assign(metadataStoreParams, options.extraMD);
            }

this gets translated (in metadataStoreObject) to this:

        if (versioning) {
            options.versioning = versioning;
        }
        if (versionId || versionId === '') {
            options.versionId = versionId;
        }

        if (deleteNullKey) {
            options.deleteNullKey = deleteNullKey;
        }

        const { isNull, nullVersionId, nullUploadId, isDeleteMarker } = params;
        md.setIsNull(isNull)
            .setNullVersionId(nullVersionId)
            .setNullUploadId(nullUploadId)
            .setIsDeleteMarker(isDeleteMarker);
        if (versionId && versionId !== 'null') {
            md.setVersionId(versionId);
        }
        if (isNull && !config.nullVersionCompatMode) {
            md.setIsNull2(true);
        }

→ so it seems many more fields may be set, to cover all cases?

[DarkIsDude]

That's good a point, thanks for pointing it. versionId;versioning;isNull; are already managed by the code before. So that don't make sense to manage them again here 🙏.

I added a the extraMD and the deleteNullKey !

[DarkIsDude]

somehow I am not sure this covers all cases - and not confident everything is fully tests: there are tests, but logic is quite intricate with the different versioning cases + different listings/... and all the flags: isNull, isNull2, ...

→ would be worth reviewing the tests to see if we miss some cases (e.g. all the combinations of versioning disabled/enabled/suspended, creating vs updating an object, with previous version/nullVersion/no previous version...). May help to look at putObject tests for inspiration on the test case? → since we have multiple backend, and the separation is sometimes not clear (or not clearly documented), we should make sure this test is run both with mongo and metadata backends: is it the case?

So I took time to review tests and path. With the condition if (versioning && !objMd) there is not a lot of path that can jump in that code. So I added some tests but I didn't find more of them. Keep in mind that I don't have an overview of all features / options