HDDS-13345. Security Token Service (STS) - S3 compatible API for temporary S3 credentials

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -2020,6 +2020,14 @@
       will be used for http authentication.
     </description>
   </property>
+  <property>
+    <name>ozone.s3g.sts.http.enabled</name>
+    <value>false</value>
+    <tag>OZONE, S3GATEWAY</tag>
+    <description>
+      The boolean which enables the Ozone S3Gateway STS endpoint.
+    </description>
+  </property>
   <property>
     <name>ozone.s3g.metrics.percentiles.intervals.seconds</name>
     <value>60</value>

hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml

@@ -94,6 +94,7 @@ services:
       - ./krb5.conf:/etc/krb5.conf
     ports:
       - 9878:9878
+      - 19878:19878
     env_file:
       - ./docker-config
     command: ["/opt/hadoop/bin/ozone","s3g", "-Dozone.om.transport.class=${OZONE_S3_OM_TRANSPORT:-org.apache.hadoop.ozone.om.protocolPB.GrpcOmTransportFactory}"]

hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config

@@ -93,13 +93,15 @@ OZONE-SITE.XML_hdds.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keyta
 
 OZONE-SITE.XML_ozone.security.http.kerberos.enabled=true
 OZONE-SITE.XML_ozone.s3g.secret.http.enabled=true
+OZONE-SITE.XML_ozone.s3g.sts.http.enabled=true
 OZONE-SITE.XML_ozone.http.filter.initializers=org.apache.hadoop.security.AuthenticationFilterInitializer
 
 OZONE-SITE.XML_ozone.om.http.auth.type=kerberos
 OZONE-SITE.XML_hdds.scm.http.auth.type=kerberos
 OZONE-SITE.XML_hdds.datanode.http.auth.type=kerberos
 OZONE-SITE.XML_ozone.s3g.http.auth.type=kerberos
 OZONE-SITE.XML_ozone.s3g.secret.http.auth.type=kerberos
+OZONE-SITE.XML_ozone.s3g.sts.http.auth.type=kerberos
 OZONE-SITE.XML_ozone.httpfs.http.auth.type=kerberos
 OZONE-SITE.XML_ozone.recon.http.auth.type=kerberos
 

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java

@@ -29,7 +29,8 @@
 import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.apache.hadoop.ozone.recon.ReconServerConfigKeys;
 import org.apache.hadoop.ozone.s3.S3GatewayConfigKeys;
-import org.apache.hadoop.ozone.s3secret.S3SecretConfigKeys;
+import org.apache.hadoop.ozone.s3web.s3secret.S3SecretConfigKeys;
+import org.apache.hadoop.ozone.s3web.s3sts.S3STSConfigKeys;
 
 /**
  * Tests if configuration constants documented in ozone-defaults.xml.
@@ -45,6 +46,7 @@ public void initializeMemberVariables() {
             ReconConfigKeys.class, ReconServerConfigKeys.class,
             S3GatewayConfigKeys.class,
             S3SecretConfigKeys.class,
+            S3STSConfigKeys.class,
             SCMHTTPServerConfig.class,
             SCMHTTPServerConfig.ConfigStrings.class,
             ScmConfig.ConfigStrings.class

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayWebAdminServer.java

@@ -29,10 +29,10 @@
 import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_WEBADMIN_HTTP_BIND_PORT_DEFAULT;
 import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_WEBADMIN_HTTP_ENABLED_KEY;
 import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL;
-import static org.apache.hadoop.ozone.s3secret.S3SecretConfigKeys.OZONE_S3G_SECRET_HTTP_AUTH_TYPE_DEFAULT;
-import static org.apache.hadoop.ozone.s3secret.S3SecretConfigKeys.OZONE_S3G_SECRET_HTTP_AUTH_TYPE_KEY;
-import static org.apache.hadoop.ozone.s3secret.S3SecretConfigKeys.OZONE_S3G_SECRET_HTTP_ENABLED_KEY;
-import static org.apache.hadoop.ozone.s3secret.S3SecretConfigKeys.OZONE_S3G_SECRET_HTTP_ENABLED_KEY_DEFAULT;
+import static org.apache.hadoop.ozone.s3web.s3secret.S3SecretConfigKeys.OZONE_S3G_SECRET_HTTP_AUTH_TYPE_DEFAULT;
+import static org.apache.hadoop.ozone.s3web.s3secret.S3SecretConfigKeys.OZONE_S3G_SECRET_HTTP_AUTH_TYPE_KEY;
+import static org.apache.hadoop.ozone.s3web.s3secret.S3SecretConfigKeys.OZONE_S3G_SECRET_HTTP_ENABLED_KEY;
+import static org.apache.hadoop.ozone.s3web.s3secret.S3SecretConfigKeys.OZONE_S3G_SECRET_HTTP_ENABLED_KEY_DEFAULT;
 import static org.apache.hadoop.security.authentication.server.AuthenticationFilter.AUTH_TYPE;
 
 import com.google.common.base.Strings;

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java

@@ -118,7 +118,6 @@ public static String createSignatureBase(
     }
     strToSign.append(signatureInfo.getDateTime()).append(NEWLINE);
     strToSign.append(credentialScope).append(NEWLINE);
-
     String canonicalRequest = buildCanonicalRequest(
         scheme,
         method,

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3secret/Application.java → hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3web/Application.java

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.ozone.s3secret;
+package org.apache.hadoop.ozone.s3web;
 
 import org.glassfish.jersey.server.ResourceConfig;
 
@@ -24,6 +24,6 @@
  */
 public class Application extends ResourceConfig {
   public Application() {
-    packages("org.apache.hadoop.ozone.s3secret");
+    packages(true, "org.apache.hadoop.ozone.s3web");
   }
 }

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3web/package-info.java

@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains the top level generic classes of s3 web gateway.
+ */
+package org.apache.hadoop.ozone.s3web;

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3secret/S3AdminEndpoint.java → hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3web/s3secret/S3AdminEndpoint.java

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.ozone.s3secret;
+package org.apache.hadoop.ozone.s3web.s3secret;
 
 import java.lang.annotation.ElementType;
 import java.lang.annotation.Retention;

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3secret/S3SecretAdminFilter.java → hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3web/s3secret/S3SecretAdminFilter.java

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.ozone.s3secret;
+package org.apache.hadoop.ozone.s3web.s3secret;
 
 import java.io.IOException;
 import java.security.Principal;

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3secret/S3SecretConfigKeys.java → hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3web/s3secret/S3SecretConfigKeys.java

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.ozone.s3secret;
+package org.apache.hadoop.ozone.s3web.s3secret;
 
 /**
  * This class contains constants for configuration keys used

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3secret/S3SecretEnabled.java → hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3web/s3secret/S3SecretEnabled.java

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.ozone.s3secret;
+package org.apache.hadoop.ozone.s3web.s3secret;
 
 import java.lang.annotation.ElementType;
 import java.lang.annotation.Retention;

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3secret/S3SecretEnabledEndpointRequestFilter.java → hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3web/s3secret/S3SecretEnabledEndpointRequestFilter.java

@@ -15,9 +15,9 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.ozone.s3secret;
+package org.apache.hadoop.ozone.s3web.s3secret;
 
-import static org.apache.hadoop.ozone.s3secret.S3SecretConfigKeys.OZONE_S3G_SECRET_HTTP_ENABLED_KEY;
+import static org.apache.hadoop.ozone.s3web.s3secret.S3SecretConfigKeys.OZONE_S3G_SECRET_HTTP_ENABLED_KEY;
 
 import java.io.IOException;
 import javax.inject.Inject;

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3secret/S3SecretEndpointBase.java → hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3web/s3secret/S3SecretEndpointBase.java

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.ozone.s3secret;
+package org.apache.hadoop.ozone.s3web.s3secret;
 
 import com.google.common.annotations.VisibleForTesting;
 import java.io.IOException;

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3secret/S3SecretManagementEndpoint.java → hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3web/s3secret/S3SecretManagementEndpoint.java

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.ozone.s3secret;
+package org.apache.hadoop.ozone.s3web.s3secret;
 
 import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
 import static javax.ws.rs.core.Response.Status.NOT_FOUND;

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3secret/S3SecretResponse.java → hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3web/s3secret/S3SecretResponse.java

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.ozone.s3secret;
+package org.apache.hadoop.ozone.s3web.s3secret;
 
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3secret/package-info.java → hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3web/s3secret/package-info.java

@@ -18,4 +18,4 @@
 /**
  * This package contains the top level generic classes of s3 secret gateway.
  */
-package org.apache.hadoop.ozone.s3secret;
+package org.apache.hadoop.ozone.s3web.s3secret;

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3web/s3sts/S3STSConfigKeys.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3web.s3sts;
+
+/**
+ * This class contains constants for configuration keys used
+ * in S3 STS endpoint.
+ */
+public final class S3STSConfigKeys {
+  public static final String OZONE_S3G_STS_HTTP_ENABLED_KEY =
+      "ozone.s3g.sts.http.enabled";
+
+  /**
+   * Never constructed.
+   */
+  private S3STSConfigKeys() {
+
+  }
+}

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3web/s3sts/S3STSEnabled.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3web.s3sts;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import javax.ws.rs.NameBinding;
+
+/**
+ * Annotation to disable S3 STS Endpoint.
+ */
+@NameBinding
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.TYPE, ElementType.METHOD})
+public @interface S3STSEnabled {
+}
+
+

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3web/s3sts/S3STSEnabledEndpointRequestFilter.java

@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3web.s3sts;
+
+import static org.apache.hadoop.ozone.s3web.s3sts.S3STSConfigKeys.OZONE_S3G_STS_HTTP_ENABLED_KEY;
+
+import java.io.IOException;
+import javax.inject.Inject;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.Provider;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+
+/**
+ * Filter that disables all endpoints annotated with {@link S3STSEnabled}.
+ * Condition is based on the value of the configuration key
+ * ozone.s3g.s3sts.http.enabled.
+ */
+@S3STSEnabled
+@Provider
+public class S3STSEnabledEndpointRequestFilter implements ContainerRequestFilter {
+  @Inject
+  private OzoneConfiguration ozoneConfiguration;
+
+  @Override
+  public void filter(ContainerRequestContext requestContext) throws IOException {
+    boolean isSTSEnabled = ozoneConfiguration.getBoolean(
+        OZONE_S3G_STS_HTTP_ENABLED_KEY, false);
+    if (!isSTSEnabled) {
+      String errorMessage = "S3 STS endpoint is disabled.";
+      String errorCode = "AccessDenied";
+      String xmlError = "<ErrorResponse xmlns=\"https://sts.amazonaws.com/doc/2011-06-15/\">" +
+          "<Error>" +
+          "<Type>Sender</Type>" +
+          "<Code>" + errorCode + "</Code>" +
+          "<Message>" + errorMessage + "</Message>" +
+          "</Error>" +
+          "<RequestId>" + requestContext.getHeaderString("x-amz-request-id") + "</RequestId>" +
+          "</ErrorResponse>";
+
+      requestContext.abortWith(Response.status(Response.Status.BAD_REQUEST)
+          .entity(xmlError)
+          .type(MediaType.APPLICATION_XML_TYPE)
+          .build());
+    }
+  }
+}