HDDS-8436. Support setSafeMode(), isFileClosed() FileSystem API

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java

@@ -269,6 +269,7 @@ public static boolean isReadOnly(
     case CancelSnapshotDiff:
     case ListSnapshotDiffJobs:
     case TransferLeadership:
+    case SetSafeMode:
       return true;
     case CreateVolume:
     case SetVolumeProperty:

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java

@@ -23,6 +23,7 @@
 import java.util.List;
 import java.util.UUID;
 
+import org.apache.hadoop.fs.SafeModeAction;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
 import org.apache.hadoop.ozone.OzoneAcl;
 import org.apache.hadoop.ozone.om.IOmMetadataReader;
@@ -1054,4 +1055,18 @@ void setTimes(OmKeyArgs keyArgs, long mtime, long atime)
       throws IOException;
 
   UUID refetchSecretKey() throws IOException;
+  /**
+   * Enter, leave, or get safe mode.
+   *
+   * @param action    One of {@link SafeModeAction} LEAVE, ENTER, GET,
+   *                  FORCE_EXIT.
+   * @param isChecked If true check only for Active metadata node /
+   *                  NameNode's status, else check first metadata node /
+   *                  NameNode's status.
+   * @throws IOException if set safe mode fails to proceed.
+   * @return true if the action is successfully accepted, otherwise false
+   * means rejected.
+   */
+  boolean setSafeMode(SafeModeAction action, boolean isChecked)
+      throws IOException;
 }

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java

@@ -26,6 +26,7 @@
 import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.fs.SafeModeAction;
 import org.apache.hadoop.hdds.annotation.InterfaceAudience;
 import org.apache.hadoop.hdds.client.ECReplicationConfig;
 import org.apache.hadoop.hdds.client.ReplicationConfig;
@@ -169,13 +170,16 @@
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenewDelegationTokenResponseProto;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RevokeS3SecretRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Secret;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SafeMode;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListResponse;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetAclRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetAclResponse;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetBucketPropertyRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetS3SecretRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetS3SecretResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetSafeModeRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetSafeModeResponse;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetTimesRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetVolumePropertyRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantAssignAdminRequest;
@@ -2331,6 +2335,38 @@ public void setTimes(OmKeyArgs args, long mtime, long atime)
     handleError(submitRequest(omRequest));
   }
 
+  @Override
+  public boolean setSafeMode(SafeModeAction action, boolean isChecked)
+      throws IOException {
+    SetSafeModeRequest setSafeModeRequest =
+        SetSafeModeRequest.newBuilder()
+            .setSafeMode(toProtoBuf(action))
+            .build();
+
+    OMRequest omRequest = createOMRequest(Type.SetSafeMode)
+        .setSetSafeModeRequest(setSafeModeRequest).build();
+
+    SetSafeModeResponse setSafeModeResponse =
+        handleError(submitRequest(omRequest)).getSetSafeModeResponse();
+    return setSafeModeResponse.getResponse();
+  }
+
+  private SafeMode toProtoBuf(SafeModeAction action) {
+    switch (action) {
+    case ENTER:
+      return SafeMode.ENTER;
+    case LEAVE:
+      return SafeMode.LEAVE;
+    case FORCE_EXIT:
+      return SafeMode.FORCE_EXIT;
+    case GET:
+      return SafeMode.GET;
+    default:
+      throw new IllegalArgumentException("Unsupported safe mode action " +
+          action);
+    }
+  }
+
   @VisibleForTesting
   public OmTransport getTransport() {
     return transport;

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java

@@ -48,6 +48,7 @@
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 
@@ -120,13 +121,15 @@ public void testRecovery() throws Exception {
     ThreadLocalRandom.current().nextBytes(data);
     stream.write(data);
     stream.hsync();
+    assertFalse(fs.isFileClosed(file));
 
     int count = 0;
     while (count++ < 15 && !fs.recoverLease(file)) {
       Thread.sleep(1000);
     }
     // The lease should have been recovered.
     assertTrue("File should be closed", fs.recoverLease(file));
+    assertTrue(fs.isFileClosed(file));
     // open it again, make sure the data is correct
     byte[] readData = new byte[1 << 20];
     try (FSDataInputStream fdis = fs.open(file)) {

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithFSO.java

@@ -20,12 +20,14 @@
 
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.LeaseRecoverable;
 import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
 import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
@@ -48,7 +50,9 @@
 import java.util.Collection;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -568,6 +572,29 @@ public void testCreateFile() throws Exception {
     }, 1000, 120000);
   }
 
+  /**
+   * Verify recoverLease() and isFileClosed() APIs.
+   * @throws Exception
+   */
+  @Test
+  public void testLeaseRecoverable() throws Exception {
+    // Create a file
+    Path parent = new Path("/d1/d2/");
+    Path source = new Path(parent, "file1");
+
+    LeaseRecoverable fs = (LeaseRecoverable)getFs();
+    FSDataOutputStream stream = getFs().create(source);
+    // file not visible yet
+    assertThrows(OMException.class, () -> fs.isFileClosed(source));
+    stream.write(1);
+    stream.hsync();
+    // file is visible and open
+    assertFalse(fs.isFileClosed(source));
+    assertTrue(fs.recoverLease(source));
+    // file is closed after lease recovery
+    assertTrue(fs.isFileClosed(source));
+  }
+
   private void verifyOMFileInfoFormat(OmKeyInfo omKeyInfo, String fileName,
       long parentID) {
     Assert.assertEquals("Wrong keyName", fileName, omKeyInfo.getKeyName());

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java

@@ -30,6 +30,8 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
 import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.SafeMode;
+import org.apache.hadoop.fs.SafeModeAction;
 import org.apache.hadoop.fs.StreamCapabilities;
 import org.apache.hadoop.fs.Trash;
 import org.apache.hadoop.fs.TrashPolicy;
@@ -2545,4 +2547,19 @@ public void testSetTimes() throws Exception {
     // verify that mtime is NOT updated as expected.
     Assert.assertEquals(mtime, fileStatus.getModificationTime());
   }
+
+  @Test
+  public void testSafeMode() throws Exception {
+    SafeMode safeModeFS = (SafeMode) fs;
+    // safe mode is off
+    assertFalse(safeModeFS.setSafeMode(SafeModeAction.GET));
+    // shutdown datanodes and restart SCM
+    cluster.shutdownHddsDatanodes();
+    cluster.restartStorageContainerManager(false);
+    // SCM should be in safe mode
+    assertTrue(safeModeFS.setSafeMode(SafeModeAction.GET));
+    // force exit safe mode and verify that it's out of safe mode.
+    safeModeFS.setSafeMode(SafeModeAction.FORCE_EXIT);
+    assertFalse(safeModeFS.setSafeMode(SafeModeAction.GET));
+  }
 }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystemWithFSO.java

@@ -19,8 +19,11 @@
 
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.LeaseRecoverable;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Ignore;
@@ -36,6 +39,8 @@
 import java.util.concurrent.TimeoutException;
 
 import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -128,13 +133,13 @@ public void testRenameDestinationParentDoesntExist() throws Exception {
         + root + "/b/c");
 
     // rename should fail and return false
-    Assert.assertFalse(getFs().rename(dir2SourcePath, destinPath));
+    assertFalse(getFs().rename(dir2SourcePath, destinPath));
     // (b) parent of dst is a file. /root_dir/file1/c
     Path filePath = new Path(getBucketPath() + root + "/file1");
     ContractTestUtils.touch(getFs(), filePath);
     Path newDestinPath = new Path(filePath, "c");
     // rename should fail and return false
-    Assert.assertFalse(getFs().rename(dir2SourcePath, newDestinPath));
+    assertFalse(getFs().rename(dir2SourcePath, newDestinPath));
   }
 
   @Test
@@ -188,7 +193,7 @@ public void testRenameDirToItsOwnSubDir() throws Exception {
       LOG.info("Created dir1 {}", subDir1);
       LOG.info("Rename op-> source:{} to destin:{}", sourceRoot, subDir1);
       //  rename should fail and return false
-      Assert.assertFalse(getFs().rename(sourceRoot, subDir1));
+      assertFalse(getFs().rename(sourceRoot, subDir1));
     } finally {
       getFs().delete(sourceRoot, true);
     }
@@ -265,4 +270,20 @@ public void testListStatusFSO() throws Exception {
     Assert.assertEquals(valueGreaterBatchSize, fileStatuses.length);
   }
 
+  @Test
+  public void testLeaseRecoverable() throws Exception {
+    // Create a file
+    final String dir = "dir1";
+    final String key = dir + "/key1";
+    final Path source = new Path(getBucketPath(), key);
+
+    LeaseRecoverable fs = (LeaseRecoverable)getFs();
+    FSDataOutputStream stream = getFs().create(source);
+    assertThrows(OMException.class, () -> fs.isFileClosed(source));
+    stream.write(1);
+    stream.hsync();
+    assertFalse(fs.isFileClosed(source));
+    assertTrue(fs.recoverLease(source));
+    assertTrue(fs.isFileClosed(source));
+  }
 }

hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto

@@ -139,6 +139,14 @@ enum Type {
   RefetchSecretKey = 121;
   ListSnapshotDiffJobs = 122;
   CancelSnapshotDiff = 123;
+  SetSafeMode = 124;
+}
+
+enum SafeMode {
+    ENTER = 1;
+    LEAVE = 2;
+    FORCE_EXIT = 3;
+    GET = 4;
 }
 
 message OMRequest {
@@ -259,9 +267,9 @@ message OMRequest {
   optional RecoverLeaseRequest              RecoverLeaseRequest            = 119;
   optional SetTimesRequest                   SetTimesRequest               = 120;
   optional RefetchSecretKeyRequest          RefetchSecretKeyRequest        = 121;
-
   optional ListSnapshotDiffJobRequest       ListSnapshotDiffJobRequest     = 122;
   optional CancelSnapshotDiffRequest       CancelSnapshotDiffRequest     = 123;
+  optional SetSafeModeRequest               SetSafeModeRequest             = 124;
 }
 
 message OMResponse {
@@ -374,9 +382,9 @@ message OMResponse {
   optional RecoverLeaseResponse              RecoverLeaseResponse          = 119;
   optional SetTimesResponse                   SetTimesResponse             = 120;
   optional RefetchSecretKeyResponse          RefetchSecretKeyResponse      = 121;
-
   optional ListSnapshotDiffJobResponse       ListSnapshotDiffJobResponse   = 122;
   optional CancelSnapshotDiffResponse       cancelSnapshotDiffResponse   = 123;
+  optional SetSafeModeResponse               SetSafeModeResponse           = 124;
 }
 
 enum Status {
@@ -1954,6 +1962,14 @@ message SetTimesRequest {
 message SetTimesResponse {
 }
 
+message SetSafeModeRequest {
+  required SafeMode safeMode    = 1;
+}
+
+message SetSafeModeResponse {
+  optional bool response = 1;
+}
+
 /**
  The OM service that takes care of Ozone namespace.
 */

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java

@@ -59,6 +59,7 @@
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.SafeModeAction;
 import org.apache.hadoop.hdds.HddsConfigKeys;
 import org.apache.hadoop.hdds.HddsUtils;
 import org.apache.hadoop.hdds.annotation.InterfaceAudience;
@@ -288,6 +289,7 @@
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SERVER_DEFAULT_REPLICATION_TYPE_KEY;
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.DETECTED_LOOP_IN_BUCKET_LINKS;
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FEATURE_NOT_ENABLED;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INTERNAL_ERROR;
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_AUTH_METHOD;
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST;
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.PERMISSION_DENIED;
@@ -4469,6 +4471,25 @@ public void setTimes(OmKeyArgs keyArgs, long mtime, long atime)
       throws IOException {
   }
 
+  @Override
+  public boolean setSafeMode(SafeModeAction action, boolean isChecked)
+      throws IOException {
+    switch (action) {
+    case ENTER:
+      throw new OMException("Enter safe mode is unsupported",
+          INTERNAL_ERROR);
+    case FORCE_EXIT:
+      return getScmClient().getContainerClient().forceExitSafeMode();
+    case GET:
+      return getScmClient().getContainerClient().inSafeMode();
+    //case LEAVE:
+    // TODO: support LEAVE in the future.
+    default:
+      throw new OMException("Unsupported safe mode action " + action,
+          INTERNAL_ERROR);
+    }
+  }
+
   /**
    * Write down Layout version of a finalized feature to DB on finalization.
    * @param lvm OMLayoutVersionManager