Cleanup created accounts and buckets after tests

.github/actions/archive-artifacts/action.yaml

@@ -42,3 +42,21 @@ runs:
             sh -c "kubectl exec -i -n ${NAMESPACE} kcat -- \
                        kcat -L -b ${KAFKA_SERVICE} -t {}  -C -o beginning -e -q -J \
                        > /tmp/artifacts/data/${STAGE}/kafka-messages-{}.log"
+    - name: Dump MongoDB
+      shell: bash
+      continue-on-error: true
+      run: |-
+        set -exu
+
+        ZENKO_MONGODB_DATABASE="${ZENKO_MONGODB_DATABASE:-zenko-database}"
+        MONGODB_ROOT_USERNAME="${MONGODB_ROOT_USERNAME:-root}"
+        MONGODB_ROOT_PASSWORD="${MONGODB_ROOT_PASSWORD:-rootpass}"
+        NAMESPACE="${NAMESPACE:-default}"
+        DUMP_DIR="/tmp/mongodb.dump"
+
+        kubectl exec -n ${NAMESPACE} data-db-mongodb-sharded-mongos-0 -- mongodump --db ${ZENKO_MONGODB_DATABASE} -u ${MONGODB_ROOT_USERNAME} -p ${MONGODB_ROOT_PASSWORD} --authenticationDatabase admin --out ${DUMP_DIR}
+
+        kubectl exec -n ${NAMESPACE} data-db-mongodb-sharded-mongos-0 -- bash -c "for bson_file in ${DUMP_DIR}/${ZENKO_MONGODB_DATABASE}/*.bson; do json_file=\"${DUMP_DIR}/\$(basename \${bson_file} .bson).json\"; bsondump --outFile \${json_file} \${bson_file}; done"
+
+        mkdir -p /tmp/artifacts/data/${STAGE}/mongodb-dump
+        kubectl cp ${NAMESPACE}/data-db-mongodb-sharded-mongos-0:${DUMP_DIR} /tmp/artifacts/data/${STAGE}/mongodb-dump

.github/scripts/end2end/configs/zenko_dr_sink.yaml

@@ -11,7 +11,7 @@ spec:
     userSecretName: mongodb-db-creds-pra
     usernameKey: mongodb-username
     passwordKey: mongodb-password
-    databaseName: "pradb"
+    databaseName: pradb
     writeConcern: "majority"
   kafka:
     managed:

.github/scripts/end2end/deploy-zenko.sh

@@ -41,7 +41,7 @@ export ZENKO_ANNOTATIONS="annotations:"
 export ZENKO_MONGODB_ENDPOINT="data-db-mongodb-sharded.default.svc.cluster.local:27017"
 export ZENKO_MONGODB_CONFIG="writeConcern: 'majority'
     enableSharding: true"
-export ZENKO_MONGODB_DATABASE="${ZENKO_MONGODB_DATABASE:-'datadb'}"
+export ZENKO_MONGODB_DATABASE="${ZENKO_MONGODB_DATABASE:-datadb}"
 
 if [ "${TIME_PROGRESSION_FACTOR}" -gt 1 ]; then
     export ZENKO_ANNOTATIONS="$ZENKO_ANNOTATIONS

.github/scripts/end2end/install-kind-dependencies.sh

@@ -21,7 +21,7 @@ MONGODB_ROOT_USERNAME=root
 MONGODB_ROOT_PASSWORD=rootpass
 MONGODB_APP_USERNAME=data
 MONGODB_APP_PASSWORD=datapass
-MONGODB_APP_DATABASE="${ZENKO_MONGODB_DATABASE:-'datadb'}"
+MONGODB_APP_DATABASE=${ZENKO_MONGODB_DATABASE:-datadb}
 MONGODB_RS_KEY=0123456789abcdef
 
 ENABLE_KEYCLOAK_HTTPS=${ENABLE_KEYCLOAK_HTTPS:-'false'}

.github/scripts/end2end/prepare-pra.sh

@@ -6,7 +6,7 @@ export MONGODB_PRA_DATABASE="${MONGODB_PRA_DATABASE:-'pradb'}"
 export ZENKO_MONGODB_DATABASE="${MONGODB_PRA_DATABASE}"
 export ZENKO_MONGODB_SECRET_NAME="mongodb-db-creds-pra"
 
-echo 'ZENKO_MONGODB_DATABASE="pradb"' >> "$GITHUB_ENV"
+echo 'ZENKO_MONGODB_DATABASE=pradb' >> "$GITHUB_ENV"
 echo 'ZENKO_MONGODB_SECRET_NAME="mongodb-db-creds-pra"' >> "$GITHUB_ENV"
 
 echo 'ZENKO_IAM_INGRESS="iam.dr.zenko.local"' >> "$GITHUB_ENV"

.github/workflows/end2end.yaml

@@ -470,7 +470,7 @@ jobs:
       - name: Deploy second Zenko for PRA
         run: bash deploy-zenko.sh end2end-pra default './configs/zenko.yaml'
         env:
-          ZENKO_MONGODB_DATABASE: "pradb"
+          ZENKO_MONGODB_DATABASE: pradb
         working-directory: ./.github/scripts/end2end
       - name: Add Keycloak pra user and assign StorageManager role
         shell: bash

tests/ctst/common/common.ts

@@ -5,7 +5,7 @@ import Zenko from 'world/Zenko';
 import { safeJsonParse } from './utils';
 import assert from 'assert';
 import { Admin, Kafka } from 'kafkajs';
-import { 
+import {
     createBucketWithConfiguration,
     putObject,
     runActionAgainstBucket,
@@ -15,6 +15,7 @@ import {
     addTransitionWorkflow,
 } from 'steps/utils/utils';
 import { ActionPermissionsType } from 'steps/bucket-policies/utils';
+import constants from './constants';
 
 setDefaultTimeout(Constants.DEFAULT_TIMEOUT);
 
@@ -31,13 +32,21 @@ export async function cleanS3Bucket(
     if (!bucketName) {
         return;
     }
+    if (world.getSaved<string>('objectLockMode') === constants.complianceRetention) {
+        // Do not try to clean a bucket with compliance retention
+        return;
+    }
+    Identity.useIdentity(IdentityEnum.ACCOUNT, world.getSaved<string>('accountName') ||
+        world.parameters.AccountName);
     world.resetCommand();
     world.addCommandParameter({ bucket: bucketName });
-    const createdObjects = world.getSaved<Map<string, string>>('createdObjects');
+    const createdObjects = world.getCreatedObjects();
     if (createdObjects !== undefined) {
         const results = await S3.listObjectVersions(world.getCommandParameters());
         const res = safeJsonParse<ListObjectVersionsOutput>(results.stdout);
-        assert(res.ok);
+        if (!res.ok) {
+            throw results;
+        }
         const versions = res.result!.Versions || [];
         const deleteMarkers = res.result!.DeleteMarkers || [];
         await Promise.all(versions.concat(deleteMarkers).map(obj => {
@@ -63,19 +72,14 @@ async function addMultipleObjects(this: Zenko, numberObjects: number,
             this.addToSaved('objectSize', sizeBytes);
         }
         if (userMD) {
-            this.addCommandParameter({ metadata: JSON.stringify(userMD) });
+            this.addToSaved('userMetadata', userMD);
         }
-        this.addToSaved('objectName', objectNameFinal);
-        this.logger.debug('Adding object', { objectName: objectNameFinal });
         lastResult = await putObject(this, objectNameFinal);
-        const createdObjects = this.getSaved<Map<string, string>>('createdObjects') || new Map<string, string>();
-        createdObjects.set(this.getSaved<string>('objectName'), this.getSaved<string>('versionId'));
-        this.addToSaved('createdObjects', createdObjects);
     }
     return lastResult;
 }
 
-async function addUserMetadataToObject(this: Zenko, objectName: string|undefined, userMD: string) {
+async function addUserMetadataToObject(this: Zenko, objectName: string | undefined, userMD: string) {
     const objName = objectName || this.getSaved<string>('objectName');
     const bucketName = this.getSaved<string>('bucketName');
     this.resetCommand();
@@ -154,7 +158,7 @@ Given('a tag on object {string} with key {string} and value {string}',
         this.resetCommand();
         this.addCommandParameter({ bucket: this.getSaved<string>('bucketName') });
         this.addCommandParameter({ key: objectName });
-        const versionId = this.getSaved<Map<string, string>>('createdObjects')?.get(objectName);
+        const versionId = this.getLatestObjectVersion(objectName);
         if (versionId) {
             this.addCommandParameter({ versionId });
         }
@@ -173,12 +177,12 @@ Then('object {string} should have the tag {string} with value {string}',
         this.resetCommand();
         this.addCommandParameter({ bucket: this.getSaved<string>('bucketName') });
         this.addCommandParameter({ key: objectName });
-        const versionId = this.getSaved<Map<string, string>>('createdObjects')?.get(objectName);
+        const versionId = this.getLatestObjectVersion(objectName);
         if (versionId) {
             this.addCommandParameter({ versionId });
         }
         await S3.getObjectTagging(this.getCommandParameters()).then(res => {
-            const parsed = safeJsonParse<{ TagSet: [{Key: string, Value: string}] | undefined }>(res.stdout);
+            const parsed = safeJsonParse<{ TagSet: [{ Key: string, Value: string }] | undefined }>(res.stdout);
             assert(parsed.result!.TagSet?.some(tag => tag.Key === tagKey && tag.Value === tagValue));
         });
     });
@@ -188,14 +192,14 @@ Then('object {string} should have the user metadata with key {string} and value
         this.resetCommand();
         this.addCommandParameter({ bucket: this.getSaved<string>('bucketName') });
         this.addCommandParameter({ key: objectName });
-        const versionId = this.getSaved<Map<string, string>>('createdObjects')?.get(objectName);
+        const versionId = this.getLatestObjectVersion(objectName);
         if (versionId) {
             this.addCommandParameter({ versionId });
         }
         const res = await S3.headObject(this.getCommandParameters());
         assert.ifError(res.stderr);
         assert(res.stdout);
-        const parsed = safeJsonParse<{ Metadata: {[key: string]: string} | undefined }>(res.stdout);
+        const parsed = safeJsonParse<{ Metadata: { [key: string]: string } | undefined }>(res.stdout);
         assert(parsed.ok);
         assert(parsed.result!.Metadata);
         assert(parsed.result!.Metadata[userMDKey]);
@@ -220,7 +224,7 @@ When('i delete object {string}', async function (this: Zenko, objectName: string
     this.resetCommand();
     this.addCommandParameter({ bucket: this.getSaved<string>('bucketName') });
     this.addCommandParameter({ key: objName });
-    const versionId = this.getSaved<Map<string, string>>('createdObjects')?.get(objName);
+    const versionId = this.getLatestObjectVersion(objName);
     if (versionId) {
         this.addCommandParameter({ versionId });
     }
@@ -251,7 +255,7 @@ Then('kafka consumed messages should not take too much place on disk', { timeout
             const kafkaAdmin = new Kafka({ brokers: [this.parameters.KafkaHosts] }).admin();
             const topics: string[] = (await kafkaAdmin.listTopics())
                 .filter(t => (t.includes(this.parameters.InstanceID) &&
-                !ignoredTopics.some(e => t.includes(e))));
+                    !ignoredTopics.some(e => t.includes(e))));
 
             const previousOffsets = await getTopicsOffsets(topics, kafkaAdmin);
 
@@ -378,21 +382,15 @@ Given('an upload size of {int} B for the object {string}', async function (
 ) {
     this.addToSaved('objectSize', size);
     if (this.getSaved<boolean>('preExistingObject')) {
-        if (objectName) {
-            this.addToSaved('objectName', objectName);
-        } else {
-            this.addToSaved('objectName', `object-${Utils.randomString()}`);
-        }
-        await putObject(this, this.getSaved<string>('objectName'));
+        await putObject(this, objectName);
     }
 });
 
 When('I PUT an object with size {int}', async function (this: Zenko, size: number) {
     if (size > 0) {
         this.addToSaved('objectSize', size);
     }
-    this.addToSaved('objectName', `object-${Utils.randomString()}`);
     const result = await addMultipleObjects.call(
-        this, 1, this.getSaved<string>('objectName'), size);
+        this, 1, `object-${Utils.randomString()}`, size);
     this.setResult(result!);
 });

tests/ctst/common/constants.ts

@@ -0,0 +1,4 @@
+export default {
+    complianceRetention: 'COMPLIANCE',
+    governanceRetention: 'GOVERNANCE',
+};

tests/ctst/common/hooks.ts

@@ -8,6 +8,10 @@ import Zenko from '../world/Zenko';
 import { Identity } from 'cli-testing';
 import { prepareQuotaScenarios, teardownQuotaScenarios } from 'steps/quotas/quotas';
 import { displayDebuggingInformation, preparePRA } from 'steps/pra';
+import {
+    cleanupAccount,
+} from './utils';
+import { cleanS3Bucket } from './common';
 
 // HTTPS should not cause any error for CTST
 process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
@@ -37,8 +41,33 @@ Before({ tags: '@Quotas', timeout: 1200000 }, async function (scenarioOptions) {
     await prepareQuotaScenarios(this as Zenko, scenarioOptions);
 });
 
+After(async function (this: Zenko, results) {
+    if (results.result?.status === 'FAILED') {
+        this.logger.warn('bucket was not cleaned for test', {
+            bucket: this.getSaved<string>('bucketName'),
+        });
+        return;
+    }
+    await cleanS3Bucket(
+        this,
+        this.getSaved<string>('bucketName'),
+    );
+});
+
 After({ tags: '@Quotas' }, async function () {
     await teardownQuotaScenarios(this as Zenko);
 });
 
+After({ tags: '@BP-ASSUME_ROLE_USER_CROSS_ACCOUNT'}, async function (this: Zenko, results) {
+    const crossAccountName = this.getSaved<string>('crossAccountName');
+
+    if (results.result?.status === 'FAILED' || !crossAccountName) {
+        this.logger.warn('cross account was not cleaned for test', {
+            crossAccountName,
+        });
+        return;
+    }
+    await cleanupAccount(this, crossAccountName);
+});
+
 export default Zenko;