Readonly mode (#9)

(cherry picked from commit 1ad9c8f)

Tweak tests

Author: kuenishi

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -1933,6 +1933,13 @@
       will be used for http authentication.
     </description>
   </property>
+  <property>
+    <name>ozone.s3g.readonly</name>
+    <value>false</value>
+    <tag>OZONE, S3GATEWAY</tag>
+    <description>Whether the S3Gateway blocks PUT/POST/DELETE methods or not.
+      Mostly used for system maintenance.</description>
+  </property>
 
   <property>
     <name>ozone.om.save.metrics.interval</name>

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClient.java

@@ -92,11 +92,19 @@ public OzoneClient(ConfigurationSource conf, ClientProtocol proxy) {
   @VisibleForTesting
   protected OzoneClient(ObjectStore objectStore,
                         ClientProtocol clientProtocol) {
+    this(objectStore, clientProtocol, new OzoneConfiguration());
+  }
+
+  @VisibleForTesting
+  protected OzoneClient(ObjectStore objectStore,
+                        ClientProtocol clientProtocol,
+                        OzoneConfiguration conf) {
     this.objectStore = objectStore;
     this.proxy = clientProtocol;
     // For the unit test
-    this.conf = new OzoneConfiguration();
+    this.conf = conf;
   }
+
   /**
    * Returns the object store associated with the Ozone Cluster.
    * @return ObjectStore

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java

@@ -44,6 +44,7 @@
 import static org.apache.hadoop.ozone.conf.OzoneServiceConfig.DEFAULT_SHUTDOWN_HOOK_PRIORITY;
 import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_KERBEROS_KEYTAB_FILE_KEY;
 import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_READONLY;
 
 /**
  * This class is used to start/stop S3 compatible rest server.
@@ -101,6 +102,8 @@ public void start() throws IOException {
     LOG.info("Starting Ozone S3 gateway");
     HddsServerUtil.initializeMetrics(ozoneConfiguration, "S3Gateway");
     jvmPauseMonitor.start();
+    LOG.info("S3 Gateway Readonly mode: {}={}", OZONE_S3G_READONLY,
+            ozoneConfiguration.get(OZONE_S3G_READONLY));
     httpServer.start();
   }
 

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java

@@ -81,6 +81,8 @@ public final class S3GatewayConfigKeys {
   public static final boolean OZONE_S3G_LIST_KEYS_SHALLOW_ENABLED_DEFAULT =
       true;
 
+  public static final String OZONE_S3G_READONLY = "ozone.s3g.readonly";
+  public static final boolean OZONE_S3G_READONLY_DEFAULT = false;
   /**
    * Never constructed.
    */

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java

@@ -69,6 +69,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 
 import static org.apache.hadoop.ozone.OzoneConsts.ETAG;
@@ -294,6 +295,12 @@ public Response put(@PathParam("bucket") String bucketName,
     long startNanos = Time.monotonicNowNanos();
     S3GAction s3GAction = S3GAction.CREATE_BUCKET;
 
+    // Check if the S3Gateway status is readonly
+    Optional<Response> checkResult = checkIfReadonly();
+    if (checkResult.isPresent()) {
+      return checkResult.get();
+    }
+
     try {
       if (aclMarker != null) {
         s3GAction = S3GAction.PUT_ACL;
@@ -401,6 +408,12 @@ public Response delete(@PathParam("bucket") String bucketName)
     long startNanos = Time.monotonicNowNanos();
     S3GAction s3GAction = S3GAction.DELETE_BUCKET;
 
+    // Check if the S3Gateway status is readonly
+    Optional<Response> checkResult = checkIfReadonly();
+    if (checkResult.isPresent()) {
+      return checkResult.get();
+    }
+
     try {
       deleteS3Bucket(bucketName);
     } catch (OMException ex) {
@@ -444,6 +457,15 @@ public MultiDeleteResponse multiDelete(@PathParam("bucket") String bucketName,
                                          MultiDeleteRequest request)
       throws OS3Exception, IOException {
     S3GAction s3GAction = S3GAction.MULTI_DELETE;
+    MultiDeleteResponse result = new MultiDeleteResponse();
+
+    // Check if the S3Gateway status is readonly
+    Optional<Response> checkResult = checkIfReadonly();
+    if (checkResult.isPresent()) {
+      Response res = checkResult.get();
+      result.addError(new Error("", res.getStatusInfo().getReasonPhrase(),
+              "The S3Gateway is in read-only mode."));
+    }
 
     OzoneBucket bucket = getBucket(bucketName);
     MultiDeleteResponse result = new MultiDeleteResponse();

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java

@@ -25,14 +25,15 @@
 import javax.ws.rs.container.ContainerRequestContext;
 import javax.ws.rs.core.Context;
 import java.io.IOException;
-import java.util.Set;
-import java.util.HashSet;
 import java.util.Arrays;
-import java.util.Map;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Collections;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -53,6 +54,7 @@
 import org.apache.hadoop.ozone.om.exceptions.OMException;
 import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
 import org.apache.hadoop.ozone.om.protocol.S3Auth;
+import org.apache.hadoop.ozone.s3.S3GatewayConfigKeys;
 import org.apache.hadoop.ozone.s3.exception.OS3Exception;
 import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
 
@@ -62,6 +64,7 @@
 import org.apache.hadoop.ozone.s3.signature.SignatureInfo;
 import org.apache.hadoop.ozone.s3.util.AuditUtils;
 import org.apache.hadoop.util.Time;
+import org.apache.http.HttpStatus;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -425,4 +428,14 @@ protected boolean isAccessDenied(OMException ex) {
         || result == ResultCodes.INVALID_TOKEN;
   }
 
+  protected Optional<Response> checkIfReadonly() {
+    // Check if the S3Gateway is in read-only mode or not.
+    if (getClient().getConfiguration().getBoolean(
+        S3GatewayConfigKeys.OZONE_S3G_READONLY,
+        S3GatewayConfigKeys.OZONE_S3G_READONLY_DEFAULT)) {
+      return Optional.of(Response.status(HttpStatus.SC_METHOD_NOT_ALLOWED).
+          header("Allow", "GET,HEAD").build());
+    }
+    return Optional.empty();
+  }
 }

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java

@@ -44,6 +44,20 @@
 import javax.ws.rs.core.StreamingOutput;
 import javax.xml.bind.DatatypeConverter;
 import org.apache.commons.io.IOUtils;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.text.ParseException;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.OptionalLong;
+
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.hdds.client.ECReplicationConfig;
@@ -220,6 +234,13 @@ public Response put(
       @QueryParam("uploadId") @DefaultValue("") String uploadID,
       final InputStream body) throws IOException, OS3Exception {
     long startNanos = Time.monotonicNowNanos();
+
+    // Check if the S3Gateway status is readonly
+    Optional<Response> checkResult = checkIfReadonly();
+    if (checkResult.isPresent()) {
+      return checkResult.get();
+    }
+
     S3GAction s3GAction = S3GAction.CREATE_KEY;
     boolean auditSuccess = true;
     PerformanceStringBuilder perf = new PerformanceStringBuilder();
@@ -676,6 +697,13 @@ public Response delete(
       @QueryParam("uploadId") @DefaultValue("") String uploadId) throws
       IOException, OS3Exception {
     long startNanos = Time.monotonicNowNanos();
+
+    // Check if the S3Gateway status is readonly
+    Optional<Response> checkResult = checkIfReadonly();
+    if (checkResult.isPresent()) {
+      return checkResult.get();
+    }
+
     S3GAction s3GAction = S3GAction.DELETE_KEY;
 
     try {
@@ -741,6 +769,13 @@ public Response initializeMultipartUpload(
   )
       throws IOException, OS3Exception {
     long startNanos = Time.monotonicNowNanos();
+
+    // Check if the S3Gateway status is readonly
+    Optional<Response> checkResult = checkIfReadonly();
+    if (checkResult.isPresent()) {
+      return checkResult.get();
+    }
+
     S3GAction s3GAction = S3GAction.INIT_MULTIPART_UPLOAD;
 
     try {
@@ -813,6 +848,13 @@ public Response completeMultipartUpload(@PathParam("bucket") String bucket,
       CompleteMultipartUploadRequest multipartUploadRequest)
       throws IOException, OS3Exception {
     long startNanos = Time.monotonicNowNanos();
+
+    // Check if the S3Gateway status is readonly
+    Optional<Response> checkResult = checkIfReadonly();
+    if (checkResult.isPresent()) {
+      return checkResult.get();
+    }
+
     S3GAction s3GAction = S3GAction.COMPLETE_MULTIPART_UPLOAD;
     OzoneVolume volume = getVolume();
     // Using LinkedHashMap to preserve ordering of parts list.

hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneClientStub.java

@@ -19,6 +19,8 @@
  */
 package org.apache.hadoop.ozone.client;
 
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+
 /**
  * In-memory OzoneClient for testing.
  */
@@ -31,6 +33,11 @@ public OzoneClientStub(ObjectStoreStub objectStoreStub) {
     super(objectStoreStub, new ClientProtocolStub(objectStoreStub));
   }
 
+  public OzoneClientStub(ObjectStoreStub objectStoreStub,
+                         OzoneConfiguration conf) {
+    super(objectStoreStub, new ClientProtocolStub(objectStoreStub), conf);
+  }
+
   @Override
   public void close() {
     //NOOP.