HDDS-6084. [Multi-Tenant] Handle upgrades to version supporting S3 multi-tenancy

hadoop-ozone/dist/src/main/compose/testlib.sh

@@ -147,8 +147,8 @@ start_docker_env(){
   create_results_dir
   export OZONE_SAFEMODE_MIN_DATANODES="${datanode_count}"
 
-  docker-compose --no-ansi down
-  if ! { docker-compose --no-ansi up -d --scale datanode="${datanode_count}" \
+  docker-compose --ansi never down
+  if ! { docker-compose --ansi never up -d --scale datanode="${datanode_count}" \
       && wait_for_safemode_exit \
       && wait_for_om_leader ; }; then
     [[ -n "$OUTPUT_NAME" ]] || OUTPUT_NAME="$COMPOSE_ENV_NAME"
@@ -235,7 +235,7 @@ execute_command_in_container(){
 ## @param       List of container names, eg datanode_1 datanode_2
 stop_containers() {
   set -e
-  docker-compose --no-ansi stop $@
+  docker-compose --ansi never stop $@
   set +e
 }
 
@@ -244,7 +244,7 @@ stop_containers() {
 ## @param       List of container names, eg datanode_1 datanode_2
 start_containers() {
   set -e
-  docker-compose --no-ansi start $@
+  docker-compose --ansi never start $@
   set +e
 }
 
@@ -280,9 +280,9 @@ wait_for_port(){
 
 ## @description  Stops a docker-compose based test environment (with saving the logs)
 stop_docker_env(){
-  docker-compose --no-ansi logs > "$RESULT_DIR/docker-$OUTPUT_NAME.log"
+  docker-compose --ansi never logs > "$RESULT_DIR/docker-$OUTPUT_NAME.log"
   if [ "${KEEP_RUNNING:-false}" = false ]; then
-     docker-compose --no-ansi down
+     docker-compose --ansi never down
   fi
 }
 

hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-config

@@ -24,8 +24,8 @@ OZONE-SITE.XML_ozone.om.address.omservice.om1=om1
 OZONE-SITE.XML_ozone.om.address.omservice.om2=om2
 OZONE-SITE.XML_ozone.om.address.omservice.om3=om3
 OZONE-SITE.XML_ozone.om.ratis.enable=true
-// setting ozone.scm.ratis.enable to false for now, as scm ha upgrade is
-// not supported yet. This is supposed to work without SCM HA configuration
+# setting ozone.scm.ratis.enable to false for now, as scm ha upgrade is
+# not supported yet. This is supposed to work without SCM HA configuration
 OZONE-SITE.XML_ozone.scm.ratis.enable=false
 OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
 OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1

hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.1.0-1.2.0/callback.sh

@@ -64,8 +64,7 @@ with_old_version_downgraded() {
 
 with_new_version_finalized() {
   _check_hdds_mlvs 2
-  # OM currently only has one layout version.
-  _check_om_mlvs 0
+  _check_om_mlvs 1
 
   validate old1
   validate new1

hadoop-ozone/dist/src/main/smoketest/upgrade/finalize.robot

@@ -19,7 +19,7 @@ Resource            ../commonlib.robot
 Test Timeout        5 minutes
 Test Setup          Run Keyword if    '${SECURITY_ENABLED}' == 'true'    Kinit test user     testuser     testuser.keytab
 
-** Test Cases ***
+*** Test Cases ***
 Finalize SCM
     ${result} =        Execute      ozone admin scm finalizeupgrade
                        #Wait Until Keyword Succeeds      3min       10sec     Should contain   ${result}   OM Preparation successful!

hadoop-ozone/dist/src/main/smoketest/upgrade/generate.robot

@@ -18,6 +18,7 @@ Documentation       Generate data
 Library             OperatingSystem
 Library             BuiltIn
 Resource            ../commonlib.robot
+Resource            ../s3/commonawslib.robot
 Test Timeout        5 minutes
 
 *** Variables ***
@@ -29,5 +30,30 @@ Create a volume, bucket and key
                         Should not contain  ${output}       Failed
     ${output} =         Execute          ozone sh bucket create /${PREFIX}-volume/${PREFIX}-bucket
                         Should not contain  ${output}       Failed
-    ${output} =         Execute          ozone sh key put /${PREFIX}-volume/${PREFIX}-bucket/${PREFIX}-key /opt/hadoop/NOTICE.txt
+                        Execute and checkrc    echo "${PREFIX}: key created using Ozone Shell" > /tmp/sourcekey    0
+    ${output} =         Execute          ozone sh key put /${PREFIX}-volume/${PREFIX}-bucket/${PREFIX}-key /tmp/sourcekey
                         Should not contain  ${output}       Failed
+                        Execute and checkrc    rm /tmp/sourcekey    0
+
+Create a bucket and key in volume s3v
+    ${output} =         Execute          ozone sh bucket create /s3v/${PREFIX}-bucket
+                        Should not contain  ${output}       Failed
+                        Execute and checkrc    echo "${PREFIX}: another key created using Ozone Shell" > /tmp/sourcekey    0
+    ${output} =         Execute          ozone sh key put /s3v/${PREFIX}-bucket/key1-shell /tmp/sourcekey
+                        Should not contain  ${output}       Failed
+                        Execute and checkrc    rm /tmp/sourcekey    0
+
+Setup credentials for S3
+    # TODO: Run "Setup secure v4 headers" instead when security is enabled
+    Run Keyword         Setup dummy credentials for S3
+
+Try to create a bucket using S3 API
+    # Note: S3 API does not return error if the bucket already exists
+    ${output} =         Create bucket with name    ${PREFIX}-bucket
+                        Should Be Equal    ${output}    ${None}
+
+Create key using S3 API
+                        Execute and checkrc    echo "${PREFIX}: key created using S3 API" > /tmp/sourcekey    0
+    ${output} =         Execute AWSS3APICli and checkrc    put-object --bucket ${PREFIX}-bucket --key key2-s3api --body /tmp/sourcekey    0
+                        Should not contain    ${output}    error
+                        Execute and checkrc    rm /tmp/sourcekey    0

hadoop-ozone/dist/src/main/smoketest/upgrade/prepare.robot

@@ -19,7 +19,7 @@ Resource            ../commonlib.robot
 Test Timeout        5 minutes
 Test Setup          Run Keyword if    '${SECURITY_ENABLED}' == 'true'    Kinit test user     testuser     testuser.keytab
 
-** Test Cases ***
+*** Test Cases ***
 Prepare Ozone Manager
     ${result} =        Execute      ozone admin om prepare -id %{OM_SERVICE_ID}
                        Wait Until Keyword Succeeds      3min       10sec     Should contain   ${result}   OM Preparation successful!

hadoop-ozone/dist/src/main/smoketest/upgrade/validate.robot

@@ -18,6 +18,7 @@ Documentation       Smoketest ozone cluster startup
 Library             OperatingSystem
 Library             BuiltIn
 Resource            ../commonlib.robot
+Resource            ../s3/commonawslib.robot
 Test Timeout        5 minutes
 
 *** Variables ***
@@ -28,3 +29,24 @@ Read data from previously created key
     ${random} =         Generate Random String  5  [NUMBERS]
     ${output} =         Execute          ozone sh key get /${PREFIX}-volume/${PREFIX}-bucket/${PREFIX}-key /tmp/key-${random}
                         Should not contain  ${output}       Failed
+    ${output} =         Execute and checkrc    cat /tmp/key-${random}    0
+                        Should contain    ${output}    ${PREFIX}: key created using Ozone Shell
+                        Execute and checkrc    rm /tmp/key-${random}    0
+
+Setup credentials for S3
+    # TODO: Run "Setup secure v4 headers" instead when security is enabled
+    Run Keyword         Setup dummy credentials for S3
+
+Read key created with Ozone Shell using S3 API
+    ${output} =         Execute AWSS3APICli and checkrc    get-object --bucket ${PREFIX}-bucket --key key1-shell /tmp/get-result    0
+                        Should contain    ${output}    "ContentLength"
+    ${output} =         Execute and checkrc    cat /tmp/get-result    0
+                        Should contain    ${output}    ${PREFIX}: another key created using Ozone Shell
+                        Execute and checkrc    rm /tmp/get-result    0
+
+Read key created with S3 API using S3 API
+    ${output} =         Execute AWSS3APICli and checkrc    get-object --bucket ${PREFIX}-bucket --key key2-s3api /tmp/get-result    0
+                        Should contain    ${output}    "ContentLength"
+    ${output} =         Execute and checkrc    cat /tmp/get-result    0
+                        Should contain    ${output}    ${PREFIX}: key created using S3 API
+                        Execute and checkrc    rm /tmp/get-result    0

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSUpgrade.java

@@ -28,7 +28,6 @@
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT;
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY;
 import static org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState.OPEN;
-import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.INITIAL_VERSION;
 import static org.apache.hadoop.ozone.upgrade.InjectedUpgradeFinalizationExecutor.UpgradeTestInjectionPoints.AFTER_COMPLETE_FINALIZATION;
 import static org.apache.hadoop.ozone.upgrade.InjectedUpgradeFinalizationExecutor.UpgradeTestInjectionPoints.AFTER_POST_FINALIZE_UPGRADE;
 import static org.apache.hadoop.ozone.upgrade.InjectedUpgradeFinalizationExecutor.UpgradeTestInjectionPoints.AFTER_PRE_FINALIZE_UPGRADE;
@@ -78,6 +77,7 @@
 import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
 import org.apache.hadoop.ozone.container.common.interfaces.Container;
 import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
+import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
 import org.apache.hadoop.ozone.upgrade.BasicUpgradeFinalizer;
 import org.apache.hadoop.ozone.upgrade.InjectedUpgradeFinalizationExecutor;
 import org.apache.hadoop.ozone.upgrade.InjectedUpgradeFinalizationExecutor.UpgradeTestInjectionPoints;
@@ -155,8 +155,9 @@ public static void initClass() {
         .setTotalPipelineNumLimit(NUM_DATA_NODES + 1)
         .setHbInterval(500)
         .setHbProcessorInterval(500)
-        .setScmLayoutVersion(INITIAL_VERSION.layoutVersion())
-        .setDnLayoutVersion(INITIAL_VERSION.layoutVersion());
+        .setOmLayoutVersion(OMLayoutFeature.INITIAL_VERSION.layoutVersion())
+        .setScmLayoutVersion(HDDSLayoutFeature.INITIAL_VERSION.layoutVersion())
+        .setDnLayoutVersion(HDDSLayoutFeature.INITIAL_VERSION.layoutVersion());
 
     // Setting the provider to a max of 100 clusters. Some of the tests here
     // use multiple clusters, so its hard to know exactly how many will be
@@ -219,7 +220,7 @@ private void createKey() throws IOException {
    * Helper function to test Pre-Upgrade conditions on the SCM
    */
   private void testPreUpgradeConditionsSCM() {
-    Assert.assertEquals(INITIAL_VERSION.layoutVersion(),
+    Assert.assertEquals(HDDSLayoutFeature.INITIAL_VERSION.layoutVersion(),
         scmVersionManager.getMetadataLayoutVersion());
     for (ContainerInfo ci : scmContainerManager.getContainers()) {
       Assert.assertEquals(HddsProtos.LifeCycleState.OPEN, ci.getState());

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java

@@ -26,13 +26,25 @@
 import org.apache.hadoop.ozone.client.rpc.RpcClient;
 import org.apache.hadoop.ozone.om.OMMultiTenantManagerImpl;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
+import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
 import org.apache.hadoop.ozone.om.protocol.S3Auth;
+import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
+import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer;
+import org.apache.ozone.test.GenericTestUtils;
+import org.apache.ozone.test.LambdaTestUtils;
+import org.apache.ozone.test.LambdaTestUtils.VoidCallable;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import java.io.IOException;
 import java.util.UUID;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.hadoop.ozone.admin.scm.FinalizeUpgradeCommandUtil.isDone;
+import static org.apache.hadoop.ozone.admin.scm.FinalizeUpgradeCommandUtil.isStarting;
 
 /**
  * Tests that S3 requests for a tenant are directed to that tenant's volume,
@@ -43,22 +55,107 @@ public class TestMultiTenantVolume {
   private static MiniOzoneCluster cluster;
   private static String s3VolumeName;
 
+  private static final String TENANT_NAME = "tenant";
+  private static final String USER_PRINCIPAL = "username";
+  private static final String BUCKET_NAME = "bucket";
+  private static final String ACCESS_ID = UUID.randomUUID().toString();
+
   @BeforeClass
   public static void initClusterProvider() throws Exception {
     OzoneConfiguration conf = new OzoneConfiguration();
     conf.setBoolean(
         OMMultiTenantManagerImpl.OZONE_OM_TENANT_DEV_SKIP_RANGER, true);
     MiniOzoneCluster.Builder builder = MiniOzoneCluster.newBuilder(conf)
-        .withoutDatanodes();
+        .withoutDatanodes()
+        .setOmLayoutVersion(OMLayoutFeature.INITIAL_VERSION.layoutVersion());
     cluster = builder.build();
     s3VolumeName = HddsClientUtils.getDefaultS3VolumeName(conf);
+
+    preFinalizationChecks(getStoreForAccessID(ACCESS_ID));
+    finalizeOMUpgrade();
   }
 
   @AfterClass
   public static void shutdownClusterProvider() {
     cluster.shutdown();
   }
 
+  private static void expectFailurePreFinalization(VoidCallable eval)
+      throws Exception {
+    LambdaTestUtils.intercept(OMException.class,
+        "cannot be invoked before finalization", eval);
+  }
+
+  /**
+   * Perform sanity checks before triggering upgrade finalization.
+   */
+  private static void preFinalizationChecks(ObjectStore store)
+      throws Exception {
+
+    // None of the tenant APIs is usable before the upgrade finalization step
+    expectFailurePreFinalization(
+        store::listTenant);
+    expectFailurePreFinalization(() ->
+        store.listUsersInTenant(TENANT_NAME, ""));
+    expectFailurePreFinalization(() ->
+        store.tenantGetUserInfo(USER_PRINCIPAL));
+    expectFailurePreFinalization(() ->
+        store.createTenant(TENANT_NAME));
+    expectFailurePreFinalization(() ->
+        store.tenantAssignUserAccessId(USER_PRINCIPAL, TENANT_NAME, ACCESS_ID));
+    expectFailurePreFinalization(() ->
+        store.tenantAssignAdmin(USER_PRINCIPAL, TENANT_NAME, true));
+    expectFailurePreFinalization(() ->
+        store.tenantRevokeAdmin(ACCESS_ID, TENANT_NAME));
+    expectFailurePreFinalization(() ->
+        store.tenantRevokeUserAccessId(ACCESS_ID));
+    expectFailurePreFinalization(() ->
+        store.deleteTenant(TENANT_NAME));
+
+    // S3 get/set/revoke secret APIs still work before finalization
+    final String accessId = "testUser1accessId1";
+    S3SecretValue s3SecretValue = store.getS3Secret(accessId);
+    Assert.assertEquals(accessId, s3SecretValue.getAwsAccessKey());
+    final String setSecret = "testsecret";
+    s3SecretValue = store.setS3Secret(accessId, setSecret);
+    Assert.assertEquals(accessId, s3SecretValue.getAwsAccessKey());
+    Assert.assertEquals(setSecret, s3SecretValue.getAwsSecret());
+    store.revokeS3Secret(accessId);
+  }
+
+  /**
+   * Trigger OM upgrade finalization from the client and block until completion
+   * (status FINALIZATION_DONE).
+   */
+  private static void finalizeOMUpgrade()
+      throws IOException, InterruptedException, TimeoutException {
+
+    // Trigger OM upgrade finalization. Ref: FinalizeUpgradeSubCommand#call
+    final OzoneManagerProtocol client = cluster.getRpcClient().getObjectStore()
+        .getClientProxy().getOzoneManagerClient();
+    final String upgradeClientID = "Test-Upgrade-Client-" + UUID.randomUUID();
+    UpgradeFinalizer.StatusAndMessages finalizationResponse =
+        client.finalizeUpgrade(upgradeClientID);
+
+    // The status should transition as soon as the client call above returns
+    Assert.assertTrue(isStarting(finalizationResponse.status()));
+
+    // Wait for the finalization to be marked as done.
+    // 10s timeout should be plenty.
+    GenericTestUtils.waitFor(() -> {
+      try {
+        final UpgradeFinalizer.StatusAndMessages progress =
+            client.queryUpgradeFinalizationProgress(
+                upgradeClientID, false, false);
+        return isDone(progress.status());
+      } catch (IOException e) {
+        Assert.fail("Unexpected exception while waiting for "
+            + "the OM upgrade to finalize: " + e.getMessage());
+      }
+      return false;
+    }, 500, 10000);
+  }
+
   @Test
   public void testDefaultS3Volume() throws Exception {
     final String bucketName = "bucket";
@@ -79,31 +176,31 @@ public void testDefaultS3Volume() throws Exception {
 
   @Test
   public void testS3TenantVolume() throws Exception {
-    final String tenant = "tenant";
-    final String principal = "username";
-    final String bucketName = "bucket";
-    final String accessID = UUID.randomUUID().toString();
 
-    ObjectStore store = getStoreForAccessID(accessID);
-    store.createTenant(tenant);
-    store.tenantAssignUserAccessId(principal, tenant, accessID);
+    ObjectStore store = getStoreForAccessID(ACCESS_ID);
+
+    store.createTenant(TENANT_NAME);
+    store.tenantAssignUserAccessId(USER_PRINCIPAL, TENANT_NAME, ACCESS_ID);
 
     // S3 volume pointed to by the store should be for the tenant.
-    Assert.assertEquals(tenant, store.getS3Volume().getName());
+    Assert.assertEquals(TENANT_NAME, store.getS3Volume().getName());
 
     // Create bucket in the tenant volume.
-    store.createS3Bucket(bucketName);
-    OzoneBucket bucket = store.getS3Bucket(bucketName);
-    Assert.assertEquals(tenant, bucket.getVolumeName());
+    store.createS3Bucket(BUCKET_NAME);
+    OzoneBucket bucket = store.getS3Bucket(BUCKET_NAME);
+    Assert.assertEquals(TENANT_NAME, bucket.getVolumeName());
 
     // A different user should not see bucket, since they will be directed to
     // the s3 volume.
     ObjectStore store2 = getStoreForAccessID(UUID.randomUUID().toString());
-    assertS3BucketNotFound(store2, bucketName);
+    assertS3BucketNotFound(store2, BUCKET_NAME);
 
     // Delete bucket.
-    store.deleteS3Bucket(bucketName);
-    assertS3BucketNotFound(store, bucketName);
+    store.deleteS3Bucket(BUCKET_NAME);
+    assertS3BucketNotFound(store, BUCKET_NAME);
+
+    store.tenantRevokeUserAccessId(ACCESS_ID);
+    store.deleteTenant(TENANT_NAME);
   }
 
   /**
@@ -112,7 +209,7 @@ public void testS3TenantVolume() throws Exception {
    * by the ObjectStore.
    */
   private void assertS3BucketNotFound(ObjectStore store, String bucketName)
-      throws Exception {
+      throws IOException {
     try {
       store.getS3Bucket(bucketName);
     } catch(OMException ex) {
@@ -131,7 +228,8 @@ private void assertS3BucketNotFound(ObjectStore store, String bucketName)
     }
   }
 
-  private ObjectStore getStoreForAccessID(String accessID) throws Exception {
+  private static ObjectStore getStoreForAccessID(String accessID)
+      throws IOException {
     // Cluster provider will modify our provided configuration. We must use
     // this version to build the client.
     OzoneConfiguration conf = cluster.getOzoneManager().getConfiguration();