From b6314d84185cffd527be00c7bd0179c307d1acd0 Mon Sep 17 00:00:00 2001 From: williamlardier Date: Wed, 25 Sep 2024 14:54:41 +0200 Subject: [PATCH 1/7] Code formatting Issue: ZENKO-4898 --- tests/ctst/common/common.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ctst/common/common.ts b/tests/ctst/common/common.ts index 5c991ab239..c3c74ed705 100644 --- a/tests/ctst/common/common.ts +++ b/tests/ctst/common/common.ts @@ -5,7 +5,7 @@ import Zenko from 'world/Zenko'; import { safeJsonParse } from './utils'; import assert from 'assert'; import { Admin, Kafka } from 'kafkajs'; -import { +import { createBucketWithConfiguration, putObject, runActionAgainstBucket, @@ -75,7 +75,7 @@ async function addMultipleObjects(this: Zenko, numberObjects: number, return lastResult; } -async function addUserMetadataToObject(this: Zenko, objectName: string|undefined, userMD: string) { +async function addUserMetadataToObject(this: Zenko, objectName: string | undefined, userMD: string) { const objName = objectName || this.getSaved('objectName'); const bucketName = this.getSaved('bucketName'); this.resetCommand(); @@ -251,7 +251,7 @@ Then('kafka consumed messages should not take too much place on disk', { timeout const kafkaAdmin = new Kafka({ brokers: [this.parameters.KafkaHosts] }).admin(); const topics: string[] = (await kafkaAdmin.listTopics()) .filter(t => (t.includes(this.parameters.InstanceID) && - !ignoredTopics.some(e => t.includes(e)))); + !ignoredTopics.some(e => t.includes(e)))); const previousOffsets = await getTopicsOffsets(topics, kafkaAdmin); From a4640622a9008cd947767d32e884d9aa23938eca Mon Sep 17 00:00:00 2001 From: williamlardier Date: Wed, 25 Sep 2024 14:58:13 +0200 Subject: [PATCH 2/7] Unify bucket creation and cleanup - Remove duplicated logic - Use a world-managed function to keep track of created objects - Clean the buckets at the end of any test, unless it failed, to ease debugging. Issue: ZENKO-4898 --- tests/ctst/common/common.ts | 67 +++++++++++----------- tests/ctst/common/hooks.ts | 14 +++++ tests/ctst/steps/bucket-policies/common.ts | 3 +- tests/ctst/steps/cloudserverAuth.ts | 8 +-- tests/ctst/steps/notifications.ts | 62 ++++++++------------ tests/ctst/steps/pra.ts | 8 +-- tests/ctst/steps/utils/utils.ts | 27 +++++---- tests/ctst/world/Zenko.ts | 19 ++++++ 8 files changed, 112 insertions(+), 96 deletions(-) diff --git a/tests/ctst/common/common.ts b/tests/ctst/common/common.ts index c3c74ed705..2055b0dfef 100644 --- a/tests/ctst/common/common.ts +++ b/tests/ctst/common/common.ts @@ -31,25 +31,33 @@ export async function cleanS3Bucket( if (!bucketName) { return; } - world.resetCommand(); - world.addCommandParameter({ bucket: bucketName }); - const createdObjects = world.getSaved>('createdObjects'); - if (createdObjects !== undefined) { - const results = await S3.listObjectVersions(world.getCommandParameters()); - const res = safeJsonParse(results.stdout); - assert(res.ok); - const versions = res.result!.Versions || []; - const deleteMarkers = res.result!.DeleteMarkers || []; - await Promise.all(versions.concat(deleteMarkers).map(obj => { - world.addCommandParameter({ key: obj.Key }); - world.addCommandParameter({ versionId: obj.VersionId }); - return S3.deleteObject(world.getCommandParameters()); - })); - world.deleteKeyFromCommand('key'); - world.deleteKeyFromCommand('versionId'); + try { + Identity.useIdentity(IdentityEnum.ACCOUNT, world.getSaved('accountName') || + world.parameters.AccountName); + world.resetCommand(); + world.addCommandParameter({ bucket: bucketName }); + const createdObjects = world.getCreatedObjects(); + if (createdObjects !== undefined) { + const results = await S3.listObjectVersions(world.getCommandParameters()); + const res = safeJsonParse(results.stdout); + if (!res.ok) { + throw results; + } + const versions = res.result!.Versions || []; + const deleteMarkers = res.result!.DeleteMarkers || []; + await Promise.all(versions.concat(deleteMarkers).map(obj => { + world.addCommandParameter({ key: obj.Key }); + world.addCommandParameter({ versionId: obj.VersionId }); + return S3.deleteObject(world.getCommandParameters()); + })); + world.deleteKeyFromCommand('key'); + world.deleteKeyFromCommand('versionId'); + } + await S3.deleteBucketLifecycle(world.getCommandParameters()); + await S3.deleteBucket(world.getCommandParameters()); + } catch (err) { + world.logger.warn('Error cleaning bucket', { bucketName, err }); } - await S3.deleteBucketLifecycle(world.getCommandParameters()); - await S3.deleteBucket(world.getCommandParameters()); } async function addMultipleObjects(this: Zenko, numberObjects: number, @@ -65,12 +73,7 @@ async function addMultipleObjects(this: Zenko, numberObjects: number, if (userMD) { this.addCommandParameter({ metadata: JSON.stringify(userMD) }); } - this.addToSaved('objectName', objectNameFinal); - this.logger.debug('Adding object', { objectName: objectNameFinal }); lastResult = await putObject(this, objectNameFinal); - const createdObjects = this.getSaved>('createdObjects') || new Map(); - createdObjects.set(this.getSaved('objectName'), this.getSaved('versionId')); - this.addToSaved('createdObjects', createdObjects); } return lastResult; } @@ -154,7 +157,7 @@ Given('a tag on object {string} with key {string} and value {string}', this.resetCommand(); this.addCommandParameter({ bucket: this.getSaved('bucketName') }); this.addCommandParameter({ key: objectName }); - const versionId = this.getSaved>('createdObjects')?.get(objectName); + const versionId = this.getLatestObjectVersion(objectName); if (versionId) { this.addCommandParameter({ versionId }); } @@ -173,7 +176,7 @@ Then('object {string} should have the tag {string} with value {string}', this.resetCommand(); this.addCommandParameter({ bucket: this.getSaved('bucketName') }); this.addCommandParameter({ key: objectName }); - const versionId = this.getSaved>('createdObjects')?.get(objectName); + const versionId = this.getLatestObjectVersion(objectName); if (versionId) { this.addCommandParameter({ versionId }); } @@ -188,7 +191,7 @@ Then('object {string} should have the user metadata with key {string} and value this.resetCommand(); this.addCommandParameter({ bucket: this.getSaved('bucketName') }); this.addCommandParameter({ key: objectName }); - const versionId = this.getSaved>('createdObjects')?.get(objectName); + const versionId = this.getLatestObjectVersion(objectName); if (versionId) { this.addCommandParameter({ versionId }); } @@ -220,7 +223,7 @@ When('i delete object {string}', async function (this: Zenko, objectName: string this.resetCommand(); this.addCommandParameter({ bucket: this.getSaved('bucketName') }); this.addCommandParameter({ key: objName }); - const versionId = this.getSaved>('createdObjects')?.get(objName); + const versionId = this.getLatestObjectVersion(objName); if (versionId) { this.addCommandParameter({ versionId }); } @@ -378,12 +381,7 @@ Given('an upload size of {int} B for the object {string}', async function ( ) { this.addToSaved('objectSize', size); if (this.getSaved('preExistingObject')) { - if (objectName) { - this.addToSaved('objectName', objectName); - } else { - this.addToSaved('objectName', `object-${Utils.randomString()}`); - } - await putObject(this, this.getSaved('objectName')); + await putObject(this, objectName); } }); @@ -391,8 +389,7 @@ When('I PUT an object with size {int}', async function (this: Zenko, size: numbe if (size > 0) { this.addToSaved('objectSize', size); } - this.addToSaved('objectName', `object-${Utils.randomString()}`); const result = await addMultipleObjects.call( - this, 1, this.getSaved('objectName'), size); + this, 1, `object-${Utils.randomString()}`, size); this.setResult(result!); }); diff --git a/tests/ctst/common/hooks.ts b/tests/ctst/common/hooks.ts index f1b70e75b9..651eb76e52 100644 --- a/tests/ctst/common/hooks.ts +++ b/tests/ctst/common/hooks.ts @@ -8,6 +8,7 @@ import Zenko from '../world/Zenko'; import { Identity } from 'cli-testing'; import { prepareQuotaScenarios, teardownQuotaScenarios } from 'steps/quotas/quotas'; import { displayDebuggingInformation, preparePRA } from 'steps/pra'; +import { cleanS3Bucket } from './common'; // HTTPS should not cause any error for CTST process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; @@ -37,6 +38,19 @@ Before({ tags: '@Quotas', timeout: 1200000 }, async function (scenarioOptions) { await prepareQuotaScenarios(this as Zenko, scenarioOptions); }); +After(async function (this: Zenko, results) { + if (results.result?.status === 'FAILED') { + this.logger.warn('bucket was not cleaned for test', { + bucket: this.getSaved('bucketName'), + }); + return; + } + await cleanS3Bucket( + this, + this.getSaved('bucketName'), + ); +}); + After({ tags: '@Quotas' }, async function () { await teardownQuotaScenarios(this as Zenko); }); diff --git a/tests/ctst/steps/bucket-policies/common.ts b/tests/ctst/steps/bucket-policies/common.ts index c9aac94364..869ccdf514 100644 --- a/tests/ctst/steps/bucket-policies/common.ts +++ b/tests/ctst/steps/bucket-policies/common.ts @@ -52,8 +52,7 @@ Given('an existing bucket prepared for the action', async function (this: Zenko) this.getSaved('withObjectLock'), this.getSaved('retentionMode')); if (this.getSaved('preExistingObject')) { - this.addToSaved('objectName', `objectforbptests-${Utils.randomString()}`); - await putObject(this, this.getSaved('objectName')); + await putObject(this, `objectforbptests-${Utils.randomString()}`); } }); diff --git a/tests/ctst/steps/cloudserverAuth.ts b/tests/ctst/steps/cloudserverAuth.ts index b4ef37dc98..a75a104be4 100644 --- a/tests/ctst/steps/cloudserverAuth.ts +++ b/tests/ctst/steps/cloudserverAuth.ts @@ -21,13 +21,13 @@ When('the user tries to perform DeleteObjects', async function (this: Zenko) { this.resetCommand(); this.useSavedIdentity(); this.addCommandParameter({ bucket: this.getSaved('bucketName') }); - const objectNames = this.getSaved('objectNameArray'); + const objects = this.getCreatedObjects(); const param: { Objects: { Key: string }[] } = { Objects: [], }; - objectNames.forEach((objectName: string) => { - param.Objects.push({ Key: objectName }); - }); + for (const key of objects) { + param.Objects.push({ Key: key[0] }); + } this.addCommandParameter({ delete: JSON.stringify(param) }); this.setResult(await S3.deleteObjects(this.getCommandParameters())); }); diff --git a/tests/ctst/steps/notifications.ts b/tests/ctst/steps/notifications.ts index 414756dfc9..a828e4b70b 100644 --- a/tests/ctst/steps/notifications.ts +++ b/tests/ctst/steps/notifications.ts @@ -1,9 +1,9 @@ -import { Then, Given, When, After } from '@cucumber/cucumber'; +import { Then, Given, When } from '@cucumber/cucumber'; import { strict as assert } from 'assert'; import { S3, Utils, KafkaHelper, AWSVersionObject, NotificationDestination } from 'cli-testing'; import { Message } from 'node-rdkafka'; -import { cleanS3Bucket } from 'common/common'; import Zenko from 'world/Zenko'; +import { putObject } from './utils/utils'; const KAFKA_TESTS_TIMEOUT = Number(process.env.KAFKA_TESTS_TIMEOUT) || 60000; @@ -38,16 +38,8 @@ interface QueueConfiguration { Events: string[]; } -async function putObject(world: Zenko) { - world.resetCommand(); - world.addCommandParameter({ bucket: world.getSaved('bucketName') }); - world.addCommandParameter({ key: world.getSaved('objectName') }); - const result = await S3.putObject(world.getCommandParameters()); - world.setResult(result); -} - -async function copyObject(world: Zenko) { - await putObject(world); +async function copyObject(world: Zenko, sourceObject: string) { + await putObject(world, sourceObject); world.resetCommand(); let objName = `object-${Utils.randomString()}`.toLocaleLowerCase(); if (world.getSaved('filterType')) { @@ -59,17 +51,17 @@ async function copyObject(world: Zenko) { world.addCommandParameter({ key: objName }); world.addCommandParameter({ copySource: - `${world.getSaved('bucketName')}/${world.getSaved('objectName') }`, + `${world.getSaved('bucketName')}/${sourceObject}`, }); world.addToSaved('objectName', objName); await S3.copyObject(world.getCommandParameters()); } -async function deleteObject(world: Zenko, putDeleteMarker = false) { - await putObject(world); +async function deleteObject(world: Zenko, objName: string, putDeleteMarker = false) { + await putObject(world, objName); world.resetCommand(); world.addCommandParameter({ bucket: world.getSaved('bucketName') }); - world.addCommandParameter({ key: world.getSaved('objectName') }); + world.addCommandParameter({ key: objName }); if (world.getSaved('bucketVersioning') !== 'Non versioned' && !putDeleteMarker) { const putResult = world.getResult(); const versionId = @@ -79,8 +71,8 @@ async function deleteObject(world: Zenko, putDeleteMarker = false) { await S3.deleteObject(world.getCommandParameters()); } -async function putTag(world: Zenko) { - await putObject(world); +async function putTag(world: Zenko, objName: string) { + await putObject(world, objName); world.resetCommand(); const tags = JSON.stringify({ TagSet: [{ @@ -89,24 +81,24 @@ async function putTag(world: Zenko) { }], }); world.addCommandParameter({ bucket: world.getSaved('bucketName') }); - world.addCommandParameter({ key: world.getSaved('objectName') }); + world.addCommandParameter({ key: objName }); world.addCommandParameter({ tagging: `'${tags}'` }); await S3.putObjectTagging(world.getCommandParameters()); } -async function deleteTag(world: Zenko) { - await putTag(world); +async function deleteTag(world: Zenko, objName: string) { + await putTag(world, objName); world.resetCommand(); world.addCommandParameter({ bucket: world.getSaved('bucketName') }); - world.addCommandParameter({ key: world.getSaved('objectName') }); + world.addCommandParameter({ key: objName }); await S3.deleteObjectTagging(world.getCommandParameters()); } -async function putAcl(world: Zenko) { - await putObject(world); +async function putAcl(world: Zenko, objName: string) { + await putObject(world, objName); world.resetCommand(); world.addCommandParameter({ bucket: world.getSaved('bucketName') }); - world.addCommandParameter({ key: world.getSaved('objectName') }); + world.addCommandParameter({ key: objName }); world.addCommandParameter({ acl: 'public-read' }); await S3.putObjectAcl(world.getCommandParameters()); } @@ -265,25 +257,25 @@ When('a {string} event is triggered {string} {string}', this.addToSaved('objectName', objName); switch (notificationType) { case 's3:ObjectCreated:Put': - await putObject(this); + await putObject(this, objName); break; case 's3:ObjectCreated:Copy': - await copyObject(this); + await copyObject(this, objName); break; case 's3:ObjectRemoved:Delete': - await deleteObject(this); + await deleteObject(this, objName); break; case 's3:ObjectTagging:Put': - await putTag(this); + await putTag(this, objName); break; case 's3:ObjectTagging:Delete': - await deleteTag(this); + await deleteTag(this, objName); break; case 's3:ObjectAcl:Put': - await putAcl(this); + await putAcl(this, objName); break; case 's3:ObjectRemoved:DeleteMarkerCreated': - await deleteObject(this, true); + await deleteObject(this, objName, true); break; default: break; @@ -336,9 +328,3 @@ Then('i should {string} a notification for {string} event in destination {int}', assert.strictEqual(receivedNotification, expected); }); -After({ tags: '@BucketNotification' }, async function (this: Zenko) { - await cleanS3Bucket( - this, - this.getSaved('bucketName'), - ); -}); diff --git a/tests/ctst/steps/pra.ts b/tests/ctst/steps/pra.ts index 3e05debddc..88f13b2a0c 100644 --- a/tests/ctst/steps/pra.ts +++ b/tests/ctst/steps/pra.ts @@ -9,10 +9,11 @@ import { getPVCFromLabel, } from './utils/kubernetes'; import { + putObject, restoreObject, verifyObjectLocation, } from 'steps/utils/utils'; -import { Constants, Identity, IdentityEnum, S3, SuperAdmin, Utils } from 'cli-testing'; +import { Constants, Identity, IdentityEnum, SuperAdmin, Utils } from 'cli-testing'; import { safeJsonParse } from 'common/utils'; import assert from 'assert'; import { EntityType } from 'world/Zenko'; @@ -298,10 +299,7 @@ When('the DATA_ACCESSOR user tries to perform PutObject on {string} site', { tim } } - this.addCommandParameter({ bucket: this.getSaved('bucketName') }); - this.addCommandParameter({ key: `${Utils.randomString()}` }); - - this.setResult(await S3.putObject(this.getCommandParameters())); + await putObject(this); }); const volumeTimeout = 60000; diff --git a/tests/ctst/steps/utils/utils.ts b/tests/ctst/steps/utils/utils.ts index 88f01b4ce2..d2eeda8260 100644 --- a/tests/ctst/steps/utils/utils.ts +++ b/tests/ctst/steps/utils/utils.ts @@ -78,8 +78,8 @@ async function runActionAgainstBucket(world: Zenko, action: string) { world.resetCommand(); world.addToSaved('ifS3Standard', true); world.addCommandParameter({ bucket: world.getSaved('bucketName') }); - if (world.getSaved('versionId')) { - world.addCommandParameter({ versionId: world.getSaved('versionId') }); + if (world.getSaved('lastVersionId')) { + world.addCommandParameter({ versionId: world.getSaved('lastVersionId') }); } // if copy object, set copy source as the saved object name, and the key as a new object name if (action === 'CopyObject') { @@ -196,18 +196,21 @@ async function createBucketWithConfiguration( } async function putObject(world: Zenko, objectName?: string) { - world.addToSaved('objectName', objectName || Utils.randomString()); - const objectNameArray = world.getSaved('objectNameArray') || []; - objectNameArray.push(world.getSaved('objectName')); - world.addToSaved('objectNameArray', objectNameArray); + world.resetCommand(); + let finalObjectName = objectName; + if (!finalObjectName) { + finalObjectName = `${Utils.randomString()}`; + } + world.addToSaved('objectName', finalObjectName); + world.logger.debug('Adding object', { objectName: finalObjectName }); await uploadSetup(world, 'PutObject'); - world.addCommandParameter({ key: world.getSaved('objectName') }); + world.addCommandParameter({ key: finalObjectName }); world.addCommandParameter({ bucket: world.getSaved('bucketName') }); const result = await S3.putObject(world.getCommandParameters()); - world.addToSaved('versionId', extractPropertyFromResults( - result, 'VersionId' - )); + const versionId = extractPropertyFromResults(result, 'VersionId'); + world.saveCreatedObject(finalObjectName, versionId || ''); await uploadTeardown(world, 'PutObject'); + world.setResult(result); return result; } @@ -291,7 +294,7 @@ async function verifyObjectLocation(this: Zenko, objectName: string, this.resetCommand(); this.addCommandParameter({ bucket: this.getSaved('bucketName') }); this.addCommandParameter({ key: objName }); - const versionId = this.getSaved>('createdObjects')?.get(objName); + const versionId = this.getLatestObjectVersion(objName); if (versionId) { this.addCommandParameter({ versionId }); } @@ -331,7 +334,7 @@ async function restoreObject(this: Zenko, objectName: string, days: number) { this.resetCommand(); this.addCommandParameter({ bucket: this.getSaved('bucketName') }); this.addCommandParameter({ key: objName }); - const versionId = this.getSaved>('createdObjects')?.get(objName); + const versionId = this.getLatestObjectVersion(objName); if (versionId) { this.addCommandParameter({ versionId }); } diff --git a/tests/ctst/world/Zenko.ts b/tests/ctst/world/Zenko.ts index ddbe4cb5f7..a682e561bb 100644 --- a/tests/ctst/world/Zenko.ts +++ b/tests/ctst/world/Zenko.ts @@ -943,6 +943,25 @@ export default class Zenko extends World { return await this.managementAPIRequest('DELETE', `/config/${this.parameters.InstanceID}/location/${locationName}`); } + + saveCreatedObject(objectName: string, versionId: string) { + const createdObjects = this.getSaved>('createdObjects') || new Map(); + createdObjects.set(objectName, (createdObjects.get(objectName) || []).concat(versionId)); + this.addToSaved('createdObjects', createdObjects); + this.addToSaved('lastVersionId', versionId); + } + + getCreatedObjects() { + return this.getSaved>('createdObjects'); + } + + getCreatedObject(objectName: string) { + return this.getSaved>('createdObjects')?.get(objectName); + } + + getLatestObjectVersion(objectName: string) { + return this.getSaved>('createdObjects')?.get(objectName)?.slice(-1)[0]; + } } setWorldConstructor(Zenko); From a781df46ef24d32692c7cf13554a5aaa1e057e24 Mon Sep 17 00:00:00 2001 From: williamlardier Date: Wed, 25 Sep 2024 14:59:48 +0200 Subject: [PATCH 3/7] Cleanup created accounts - Assume role tests create additional accounts that we must clean at the end of any (successful) scenario. - This ensures that the GetRolesForWebIdentity calls are not impacted by the high number of accounts. Issue: ZENKO-4898 --- tests/ctst/common/common.ts | 4 +- tests/ctst/common/hooks.ts | 15 ++++ tests/ctst/common/utils.ts | 161 +++++++++++++++++++++++++++++++++++- tests/ctst/world/Zenko.ts | 8 ++ 4 files changed, 184 insertions(+), 4 deletions(-) diff --git a/tests/ctst/common/common.ts b/tests/ctst/common/common.ts index 2055b0dfef..9d5bb95427 100644 --- a/tests/ctst/common/common.ts +++ b/tests/ctst/common/common.ts @@ -181,7 +181,7 @@ Then('object {string} should have the tag {string} with value {string}', this.addCommandParameter({ versionId }); } await S3.getObjectTagging(this.getCommandParameters()).then(res => { - const parsed = safeJsonParse<{ TagSet: [{Key: string, Value: string}] | undefined }>(res.stdout); + const parsed = safeJsonParse<{ TagSet: [{ Key: string, Value: string }] | undefined }>(res.stdout); assert(parsed.result!.TagSet?.some(tag => tag.Key === tagKey && tag.Value === tagValue)); }); }); @@ -198,7 +198,7 @@ Then('object {string} should have the user metadata with key {string} and value const res = await S3.headObject(this.getCommandParameters()); assert.ifError(res.stderr); assert(res.stdout); - const parsed = safeJsonParse<{ Metadata: {[key: string]: string} | undefined }>(res.stdout); + const parsed = safeJsonParse<{ Metadata: { [key: string]: string } | undefined }>(res.stdout); assert(parsed.ok); assert(parsed.result!.Metadata); assert(parsed.result!.Metadata[userMDKey]); diff --git a/tests/ctst/common/hooks.ts b/tests/ctst/common/hooks.ts index 651eb76e52..ed19aea392 100644 --- a/tests/ctst/common/hooks.ts +++ b/tests/ctst/common/hooks.ts @@ -8,6 +8,9 @@ import Zenko from '../world/Zenko'; import { Identity } from 'cli-testing'; import { prepareQuotaScenarios, teardownQuotaScenarios } from 'steps/quotas/quotas'; import { displayDebuggingInformation, preparePRA } from 'steps/pra'; +import { + cleanupAccount, +} from './utils'; import { cleanS3Bucket } from './common'; // HTTPS should not cause any error for CTST @@ -55,4 +58,16 @@ After({ tags: '@Quotas' }, async function () { await teardownQuotaScenarios(this as Zenko); }); +After({ tags: '@BP-ASSUME_ROLE_USER_CROSS_ACCOUNT'}, async function (this: Zenko, results) { + const crossAccountName = this.getSaved('crossAccountName'); + + if (results.result?.status === 'FAILED' || !crossAccountName) { + this.logger.warn('cross account was not cleaned for test', { + crossAccountName, + }); + return; + } + await cleanupAccount(this, crossAccountName); +}); + export default Zenko; diff --git a/tests/ctst/common/utils.ts b/tests/ctst/common/utils.ts index 407e33cc62..a6dfee6fda 100644 --- a/tests/ctst/common/utils.ts +++ b/tests/ctst/common/utils.ts @@ -1,9 +1,16 @@ import { exec } from 'child_process'; import http from 'http'; import { createHash } from 'crypto'; +import { Command, IAM, Identity, IdentityEnum } from 'cli-testing'; import { - Command, -} from 'cli-testing'; + AttachedPolicy, + Group, + Policy, + Role, + User, +} from '@aws-sdk/client-iam'; +import { AWSCliOptions } from 'cli-testing'; +import Zenko from 'world/Zenko'; /** * This helper will dynamically extract a property from a CLI result @@ -140,3 +147,153 @@ export async function request(options: http.RequestOptions, data: string | undef export function hashStringAndKeepFirst20Characters(input: string) { return createHash('sha256').update(input).digest('hex').slice(0, 20); } + +export async function listAllEntities( + listFn: (params: AWSCliOptions) => Promise, + responseKey: string, +): Promise { + let marker; + const allEntities: T[] = []; + let parsedResponse; + do { + const response = await listFn({ marker }); + if (response.err) { + throw new Error(response.err); + } + parsedResponse = JSON.parse(response.stdout); + const entities = parsedResponse[responseKey] || []; + entities.forEach((entity: T) => { + if (entity.Path?.includes('/scality-internal/')) { + return; + } + allEntities.push(entity); + }); + marker = parsedResponse.Marker; + } while (parsedResponse.IsTruncated); + return allEntities; +}; + +export async function listAttachedPolicies( + listFn: (params: AWSCliOptions) => Promise, +): Promise { + let marker; + const allPolicies: T[] = []; + let parsedResponse; + do { + const response = await listFn({ marker }); + if (response.err) { + throw new Error(response.err); + } + parsedResponse = JSON.parse(response.stdout); + const policies = parsedResponse.AttachedPolicies || []; + policies.forEach((policy: T) => { + if (policy.PolicyArn?.includes('/scality-internal/')) { + return; + } + allPolicies.push(policy); + }); + marker = parsedResponse.Marker; + } while (parsedResponse.IsTruncated); + return allPolicies; +} + +export async function cleanupAccount(world: Zenko, accountName: string) { + try { + await world.deleteAccount(accountName); + } catch (err) { + world.logger?.debug('Account has attached resources',{ + accountName, + err, + }); + } + + try { + Identity.useIdentity(IdentityEnum.ACCOUNT, accountName); + + // List and detach policies for each user + const allUsers = await listAllEntities(IAM.listUsers, 'Users'); + for (const user of allUsers) { + const allUserPolicies = await listAttachedPolicies( + params => IAM.listAttachedUserPolicies({ userName: user.UserName, ...params }), + ); + for (const policy of allUserPolicies) { + const result = await IAM.detachUserPolicy({ + userName: user.UserName, policyArn: policy.PolicyArn }); + if (result.err) { + throw new Error(result.err); + } + } + } + + // List and detach policies for each group + const allGroups = await listAllEntities(IAM.listGroups, 'Groups'); + for (const group of allGroups) { + const allGroupPolicies = await listAttachedPolicies( + params => IAM.listAttachedGroupPolicies({ groupName: group.GroupName, ...params }), + ); + for (const policy of allGroupPolicies) { + const result = await IAM.detachGroupPolicy({ + groupName: group.GroupName, policyArn: policy.PolicyArn }); + if (result.err) { + throw new Error(result.err); + } + } + } + + // List and detach policies for each role + const allRoles = await listAllEntities(IAM.listRoles, 'Roles'); + for (const role of allRoles) { + const allRolePolicies = await listAttachedPolicies( + params => IAM.listAttachedRolePolicies({ roleName: role.RoleName, ...params }), + ); + for (const policy of allRolePolicies) { + const result = await IAM.detachRolePolicy({ + roleName: role.RoleName, policyArn: policy.PolicyArn }); + if (result.err) { + throw new Error(result.err); + } + } + } + + // Delete all policies + const allPolicies = await listAllEntities(IAM.listPolicies, 'Policies'); + for (const policy of allPolicies) { + const result = await IAM.deletePolicy({ policyArn: policy.Arn }); + if (result.err) { + throw new Error(result.err); + } + } + + // Delete all roles + for (const role of allRoles) { + const result = await IAM.deleteRole({ roleName: role.RoleName }); + if (result.err) { + throw new Error(result.err); + } + } + + // Delete all groups + for (const group of allGroups) { + const result = await IAM.deleteGroup({ groupName: group.GroupName }); + if (result.err) { + throw new Error(result.err); + } + } + + // Delete all users + for (const user of allUsers) { + const result = await IAM.deleteUser({ userName: user.UserName }); + if (result.err) { + throw new Error(result.err); + } + } + + // Finally, delete the account + await world.deleteAccount(accountName); + } catch (err) { + world.logger.warn('Error while deleting cross account', { + accountName, + error: err, + }); + } +} diff --git a/tests/ctst/world/Zenko.ts b/tests/ctst/world/Zenko.ts index a682e561bb..7e097ea50b 100644 --- a/tests/ctst/world/Zenko.ts +++ b/tests/ctst/world/Zenko.ts @@ -442,6 +442,13 @@ export default class Zenko extends World { this.saveIdentityInformation(accountName, IdentityEnum.ACCOUNT, accountName); } + async deleteAccount(name: string) { + if (!name) { + throw new Error('No account name provided'); + } + await SuperAdmin.deleteAccount({ accountName: name }); + } + /** * Creates an assumed role session with a duration of 12 hours. * @param {boolean} crossAccount - If true, the role will be assumed cross account. @@ -476,6 +483,7 @@ export default class Zenko extends World { }); Identity.addIdentity(IdentityEnum.ACCOUNT, account2.account.name, account2Credentials, undefined, true); + this.addToSaved('crossAccountName', account2.account.name); accountToBeAssumedFrom = account2.account.name; } From 9bbca112f9b6c16e073b9d17f981ebe5b7176fa9 Mon Sep 17 00:00:00 2001 From: williamlardier Date: Wed, 25 Sep 2024 15:00:25 +0200 Subject: [PATCH 4/7] Simplify and unify pradb name - Quotas are not needed, plus, they cause errors when using the env variable in a kubectl command. Issue: ZENKO-4898 --- .github/scripts/end2end/configs/zenko_dr_sink.yaml | 2 +- .github/scripts/end2end/deploy-zenko.sh | 2 +- .github/scripts/end2end/install-kind-dependencies.sh | 2 +- .github/scripts/end2end/prepare-pra.sh | 2 +- .github/workflows/end2end.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/scripts/end2end/configs/zenko_dr_sink.yaml b/.github/scripts/end2end/configs/zenko_dr_sink.yaml index 1fb9ea103c..bf4c923555 100644 --- a/.github/scripts/end2end/configs/zenko_dr_sink.yaml +++ b/.github/scripts/end2end/configs/zenko_dr_sink.yaml @@ -11,7 +11,7 @@ spec: userSecretName: mongodb-db-creds-pra usernameKey: mongodb-username passwordKey: mongodb-password - databaseName: "pradb" + databaseName: pradb writeConcern: "majority" kafka: managed: diff --git a/.github/scripts/end2end/deploy-zenko.sh b/.github/scripts/end2end/deploy-zenko.sh index caaf418ebe..b62e665112 100755 --- a/.github/scripts/end2end/deploy-zenko.sh +++ b/.github/scripts/end2end/deploy-zenko.sh @@ -41,7 +41,7 @@ export ZENKO_ANNOTATIONS="annotations:" export ZENKO_MONGODB_ENDPOINT="data-db-mongodb-sharded.default.svc.cluster.local:27017" export ZENKO_MONGODB_CONFIG="writeConcern: 'majority' enableSharding: true" -export ZENKO_MONGODB_DATABASE="${ZENKO_MONGODB_DATABASE:-'datadb'}" +export ZENKO_MONGODB_DATABASE="${ZENKO_MONGODB_DATABASE:-datadb}" if [ "${TIME_PROGRESSION_FACTOR}" -gt 1 ]; then export ZENKO_ANNOTATIONS="$ZENKO_ANNOTATIONS diff --git a/.github/scripts/end2end/install-kind-dependencies.sh b/.github/scripts/end2end/install-kind-dependencies.sh index afc63da04a..23470e45a2 100755 --- a/.github/scripts/end2end/install-kind-dependencies.sh +++ b/.github/scripts/end2end/install-kind-dependencies.sh @@ -21,7 +21,7 @@ MONGODB_ROOT_USERNAME=root MONGODB_ROOT_PASSWORD=rootpass MONGODB_APP_USERNAME=data MONGODB_APP_PASSWORD=datapass -MONGODB_APP_DATABASE="${ZENKO_MONGODB_DATABASE:-'datadb'}" +MONGODB_APP_DATABASE=${ZENKO_MONGODB_DATABASE:-datadb} MONGODB_RS_KEY=0123456789abcdef ENABLE_KEYCLOAK_HTTPS=${ENABLE_KEYCLOAK_HTTPS:-'false'} diff --git a/.github/scripts/end2end/prepare-pra.sh b/.github/scripts/end2end/prepare-pra.sh index f2b82a203c..bd49bf5296 100644 --- a/.github/scripts/end2end/prepare-pra.sh +++ b/.github/scripts/end2end/prepare-pra.sh @@ -6,7 +6,7 @@ export MONGODB_PRA_DATABASE="${MONGODB_PRA_DATABASE:-'pradb'}" export ZENKO_MONGODB_DATABASE="${MONGODB_PRA_DATABASE}" export ZENKO_MONGODB_SECRET_NAME="mongodb-db-creds-pra" -echo 'ZENKO_MONGODB_DATABASE="pradb"' >> "$GITHUB_ENV" +echo 'ZENKO_MONGODB_DATABASE=pradb' >> "$GITHUB_ENV" echo 'ZENKO_MONGODB_SECRET_NAME="mongodb-db-creds-pra"' >> "$GITHUB_ENV" echo 'ZENKO_IAM_INGRESS="iam.dr.zenko.local"' >> "$GITHUB_ENV" diff --git a/.github/workflows/end2end.yaml b/.github/workflows/end2end.yaml index 7e985d0eee..d1053a659b 100644 --- a/.github/workflows/end2end.yaml +++ b/.github/workflows/end2end.yaml @@ -470,7 +470,7 @@ jobs: - name: Deploy second Zenko for PRA run: bash deploy-zenko.sh end2end-pra default './configs/zenko.yaml' env: - ZENKO_MONGODB_DATABASE: "pradb" + ZENKO_MONGODB_DATABASE: pradb working-directory: ./.github/scripts/end2end - name: Add Keycloak pra user and assign StorageManager role shell: bash From 7ab8dce2f70277f432bfa0d6c48d323336cb4ba6 Mon Sep 17 00:00:00 2001 From: williamlardier Date: Thu, 26 Sep 2024 10:37:07 +0200 Subject: [PATCH 5/7] Dump mongodb at the end of the tests - Tests should cleanup resources, unless a scenario failed - Use of mongodump and bsondump for fast speed Issue: ZENKO-4898 --- .github/actions/archive-artifacts/action.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/actions/archive-artifacts/action.yaml b/.github/actions/archive-artifacts/action.yaml index 0f18c7e653..47aad4a4fc 100644 --- a/.github/actions/archive-artifacts/action.yaml +++ b/.github/actions/archive-artifacts/action.yaml @@ -42,3 +42,21 @@ runs: sh -c "kubectl exec -i -n ${NAMESPACE} kcat -- \ kcat -L -b ${KAFKA_SERVICE} -t {} -C -o beginning -e -q -J \ > /tmp/artifacts/data/${STAGE}/kafka-messages-{}.log" + - name: Dump MongoDB + shell: bash + continue-on-error: true + run: |- + set -exu + + ZENKO_MONGODB_DATABASE="${ZENKO_MONGODB_DATABASE:-zenko-database}" + MONGODB_ROOT_USERNAME="${MONGODB_ROOT_USERNAME:-root}" + MONGODB_ROOT_PASSWORD="${MONGODB_ROOT_PASSWORD:-rootpass}" + NAMESPACE="${NAMESPACE:-default}" + DUMP_DIR="/tmp/mongodb.dump" + + kubectl exec -n ${NAMESPACE} data-db-mongodb-sharded-mongos-0 -- mongodump --db ${ZENKO_MONGODB_DATABASE} -u ${MONGODB_ROOT_USERNAME} -p ${MONGODB_ROOT_PASSWORD} --authenticationDatabase admin --out ${DUMP_DIR} + + kubectl exec -n ${NAMESPACE} data-db-mongodb-sharded-mongos-0 -- bash -c "for bson_file in ${DUMP_DIR}/${ZENKO_MONGODB_DATABASE}/*.bson; do json_file=\"${DUMP_DIR}/\$(basename \${bson_file} .bson).json\"; bsondump --outFile \${json_file} \${bson_file}; done" + + mkdir -p /tmp/artifacts/data/${STAGE}/mongodb-dump + kubectl cp ${NAMESPACE}/data-db-mongodb-sharded-mongos-0:${DUMP_DIR} /tmp/artifacts/data/${STAGE}/mongodb-dump From 2d6e5d88c39f91563fe9862d4a0925287e8be0ad Mon Sep 17 00:00:00 2001 From: williamlardier Date: Thu, 3 Oct 2024 14:00:46 +0200 Subject: [PATCH 6/7] Throw when bucket cannot be cleaned - Any bucket with Compliance retention is ignored - Governance is accepted as we delete the bucket with the account identity. Issue: ZENKO-4898 --- tests/ctst/common/common.ts | 51 ++++++++++++++++--------------- tests/ctst/common/constants.ts | 4 +++ tests/ctst/steps/notifications.ts | 1 - tests/ctst/steps/sosapi.ts | 1 - tests/ctst/steps/utils/utils.ts | 4 ++- 5 files changed, 33 insertions(+), 28 deletions(-) create mode 100644 tests/ctst/common/constants.ts diff --git a/tests/ctst/common/common.ts b/tests/ctst/common/common.ts index 9d5bb95427..abedad6dad 100644 --- a/tests/ctst/common/common.ts +++ b/tests/ctst/common/common.ts @@ -15,6 +15,7 @@ import { addTransitionWorkflow, } from 'steps/utils/utils'; import { ActionPermissionsType } from 'steps/bucket-policies/utils'; +import constants from './constants'; setDefaultTimeout(Constants.DEFAULT_TIMEOUT); @@ -31,33 +32,33 @@ export async function cleanS3Bucket( if (!bucketName) { return; } - try { - Identity.useIdentity(IdentityEnum.ACCOUNT, world.getSaved('accountName') || - world.parameters.AccountName); - world.resetCommand(); - world.addCommandParameter({ bucket: bucketName }); - const createdObjects = world.getCreatedObjects(); - if (createdObjects !== undefined) { - const results = await S3.listObjectVersions(world.getCommandParameters()); - const res = safeJsonParse(results.stdout); - if (!res.ok) { - throw results; - } - const versions = res.result!.Versions || []; - const deleteMarkers = res.result!.DeleteMarkers || []; - await Promise.all(versions.concat(deleteMarkers).map(obj => { - world.addCommandParameter({ key: obj.Key }); - world.addCommandParameter({ versionId: obj.VersionId }); - return S3.deleteObject(world.getCommandParameters()); - })); - world.deleteKeyFromCommand('key'); - world.deleteKeyFromCommand('versionId'); + if (world.getSaved('objectLockMode') === constants.complianceRetention) { + // Do not try to clean a bucket with compliance retention + return; + } + Identity.useIdentity(IdentityEnum.ACCOUNT, world.getSaved('accountName') || + world.parameters.AccountName); + world.resetCommand(); + world.addCommandParameter({ bucket: bucketName }); + const createdObjects = world.getCreatedObjects(); + if (createdObjects !== undefined) { + const results = await S3.listObjectVersions(world.getCommandParameters()); + const res = safeJsonParse(results.stdout); + if (!res.ok) { + throw results; } - await S3.deleteBucketLifecycle(world.getCommandParameters()); - await S3.deleteBucket(world.getCommandParameters()); - } catch (err) { - world.logger.warn('Error cleaning bucket', { bucketName, err }); + const versions = res.result!.Versions || []; + const deleteMarkers = res.result!.DeleteMarkers || []; + await Promise.all(versions.concat(deleteMarkers).map(obj => { + world.addCommandParameter({ key: obj.Key }); + world.addCommandParameter({ versionId: obj.VersionId }); + return S3.deleteObject(world.getCommandParameters()); + })); + world.deleteKeyFromCommand('key'); + world.deleteKeyFromCommand('versionId'); } + await S3.deleteBucketLifecycle(world.getCommandParameters()); + await S3.deleteBucket(world.getCommandParameters()); } async function addMultipleObjects(this: Zenko, numberObjects: number, diff --git a/tests/ctst/common/constants.ts b/tests/ctst/common/constants.ts new file mode 100644 index 0000000000..d6d07f45b4 --- /dev/null +++ b/tests/ctst/common/constants.ts @@ -0,0 +1,4 @@ +export default { + complianceRetention: 'COMPLIANCE', + governanceRetention: 'GOVERNANCE', +}; diff --git a/tests/ctst/steps/notifications.ts b/tests/ctst/steps/notifications.ts index a828e4b70b..e2c802f7ef 100644 --- a/tests/ctst/steps/notifications.ts +++ b/tests/ctst/steps/notifications.ts @@ -296,7 +296,6 @@ Then('notifications should be enabled for {string} event in destination {int}', (this.getSaved('notificationDestinations')[destination]).destinationName; }) as QueueConfiguration; assert(destinationConfiguration.Events.includes(notificationType)); - await S3.deleteBucket(this.getCommandParameters()); }); Then('i should {string} a notification for {string} event in destination {int}', diff --git a/tests/ctst/steps/sosapi.ts b/tests/ctst/steps/sosapi.ts index ebc72577dd..bc4e223e24 100644 --- a/tests/ctst/steps/sosapi.ts +++ b/tests/ctst/steps/sosapi.ts @@ -71,6 +71,5 @@ Then('the request should be {string}', async function (this: Zenko, result: stri const decision = this.checkResults([this.getResult()]); assert.strictEqual(decision, result === 'accepted'); this.addCommandParameter({ bucket: this.getSaved('bucketName') }); - await S3.deleteBucket(this.getCommandParameters()); await deleteFile(this.getSaved('tempFileName')); }); diff --git a/tests/ctst/steps/utils/utils.ts b/tests/ctst/steps/utils/utils.ts index d2eeda8260..19a6941e6d 100644 --- a/tests/ctst/steps/utils/utils.ts +++ b/tests/ctst/steps/utils/utils.ts @@ -12,6 +12,7 @@ import { import { extractPropertyFromResults, s3FunctionExtraParams, safeJsonParse } from 'common/utils'; import Zenko from 'world/Zenko'; import assert from 'assert'; +import constants from 'common/constants'; enum AuthorizationType { ALLOW = 'Allow', @@ -181,7 +182,8 @@ async function createBucketWithConfiguration( world.addCommandParameter({ versioningConfiguration: 'Status=Enabled' }); await S3.putBucketVersioning(world.getCommandParameters()); } - if (retentionMode === 'GOVERNANCE' || retentionMode === 'COMPLIANCE') { + if (retentionMode === constants.governanceRetention || retentionMode === constants.complianceRetention) { + world.addToSaved('objectLockMode', retentionMode); world.resetCommand(); world.addCommandParameter({ bucket: usedBucketName }); world.addCommandParameter({ From 9661f41c0c68e7bc87104e66ab22d1dc63d1835d Mon Sep 17 00:00:00 2001 From: williamlardier Date: Thu, 3 Oct 2024 14:21:06 +0200 Subject: [PATCH 7/7] Handle user metadata and body cleanup Issue: ZENKO-4898 --- tests/ctst/common/common.ts | 2 +- tests/ctst/steps/utils/utils.ts | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/ctst/common/common.ts b/tests/ctst/common/common.ts index abedad6dad..2b3f40998d 100644 --- a/tests/ctst/common/common.ts +++ b/tests/ctst/common/common.ts @@ -72,7 +72,7 @@ async function addMultipleObjects(this: Zenko, numberObjects: number, this.addToSaved('objectSize', sizeBytes); } if (userMD) { - this.addCommandParameter({ metadata: JSON.stringify(userMD) }); + this.addToSaved('userMetadata', userMD); } lastResult = await putObject(this, objectNameFinal); } diff --git a/tests/ctst/steps/utils/utils.ts b/tests/ctst/steps/utils/utils.ts index 19a6941e6d..8f8ec9bc36 100644 --- a/tests/ctst/steps/utils/utils.ts +++ b/tests/ctst/steps/utils/utils.ts @@ -47,6 +47,7 @@ async function uploadSetup(world: Zenko, action: string) { world.addCommandParameter({ body: world.getSaved('tempFileName') }); } } + async function uploadTeardown(world: Zenko, action: string) { if (action !== 'PutObject' && action !== 'UploadPart') { return; @@ -54,6 +55,7 @@ async function uploadTeardown(world: Zenko, action: string) { const objectSize = world.getSaved('objectSize') || 0; if (objectSize > 0) { await deleteFile(world.getSaved('tempFileName')); + world.deleteKeyFromCommand('body'); } } @@ -208,6 +210,10 @@ async function putObject(world: Zenko, objectName?: string) { await uploadSetup(world, 'PutObject'); world.addCommandParameter({ key: finalObjectName }); world.addCommandParameter({ bucket: world.getSaved('bucketName') }); + const userMetadata = world.getSaved('userMetadata'); + if (userMetadata) { + world.addCommandParameter({ metadata: JSON.stringify(userMetadata) }); + } const result = await S3.putObject(world.getCommandParameters()); const versionId = extractPropertyFromResults(result, 'VersionId'); world.saveCreatedObject(finalObjectName, versionId || '');