HDDS-1299. Support TokenIssuer interface for running jobs with OzoneFileSystem.

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java

@@ -19,13 +19,15 @@
 package org.apache.hadoop.ozone.client;
 
 import java.io.IOException;
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 import java.util.NoSuchElementException;
 import java.util.Objects;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
 import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
 import org.apache.hadoop.hdds.tracing.TracingUtil;
 import org.apache.hadoop.io.Text;
@@ -50,6 +52,7 @@ public class ObjectStore {
    * The proxy used for connecting to the cluster and perform
    * client operations.
    */
+  // TODO: remove rest api and client
   private final ClientProtocol proxy;
 
   /**
@@ -259,6 +262,14 @@ public void deleteVolume(String volumeName) throws IOException {
     proxy.deleteVolume(volumeName);
   }
 
+  public KeyProvider getKeyProvider() throws IOException {
+    return proxy.getKeyProvider();
+  }
+
+  public URI getKeyProviderUri() throws IOException {
+    return proxy.getKeyProviderUri();
+  }
+
   /**
    * An Iterator to iterate over {@link OzoneVolume} list.
    */
@@ -426,5 +437,11 @@ public void cancelDelegationToken(Token<OzoneTokenIdentifier> token)
     proxy.cancelDelegationToken(token);
   }
 
+  /**
+   * @return canonical service name of ozone delegation token.
+   */
+  public String getCanonicalServiceName() {
+    return proxy.getCanonicalServiceName();
+  }
 
 }

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.ozone.client.protocol;
 
 import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.crypto.key.KeyProvider;
 import org.apache.hadoop.hdds.protocol.StorageType;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.ozone.OzoneAcl;
@@ -34,6 +35,7 @@
 import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
 
 import java.io.IOException;
+import java.net.URI;
 import java.util.List;
 import java.util.Map;
 
@@ -511,4 +513,24 @@ void cancelDelegationToken(Token<OzoneTokenIdentifier> token)
 
   @VisibleForTesting
   OMFailoverProxyProvider getOMProxyProvider();
+
+  /**
+   * Get KMS client provider.
+   * @return KMS client provider.
+   * @throws IOException
+   */
+  KeyProvider getKeyProvider() throws IOException;
+
+  /**
+   * Get KMS client provider uri.
+   * @return KMS client provider uri.
+   * @throws IOException
+   */
+  URI getKeyProviderUri() throws IOException;
+
+  /**
+   * Get CanonicalServiceName for ozone delegation token.
+   * @return Canonical Service Name of ozone delegation token.
+   */
+  String getCanonicalServiceName();
 }

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java

@@ -23,6 +23,7 @@
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
 import org.apache.hadoop.hdds.protocol.StorageType;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
@@ -42,6 +43,7 @@
 import org.apache.hadoop.ozone.client.rest.response.BucketInfo;
 import org.apache.hadoop.ozone.client.rest.response.KeyInfoDetails;
 import org.apache.hadoop.ozone.client.rest.response.VolumeInfo;
+import org.apache.hadoop.ozone.client.rpc.OzoneKMSUtil;
 import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.apache.hadoop.ozone.om.ha.OMFailoverProxyProvider;
 import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
@@ -729,6 +731,17 @@ public OMFailoverProxyProvider getOMProxyProvider() {
     return null;
   }
 
+  @Override
+  public KeyProvider getKeyProvider() throws IOException {
+    // TODO: fix me to support kms instances for difference OMs
+    return OzoneKMSUtil.getKeyProvider(conf, getKeyProviderUri());
+  }
+
+  @Override
+  public URI getKeyProviderUri() throws IOException {
+    return OzoneKMSUtil.getKeyProviderUri(ugi, null, null, conf);
+  }
+
   @Override
   public OzoneInputStream getKey(
       String volumeName, String bucketName, String keyName)
@@ -1060,4 +1073,13 @@ public OzoneMultipartUploadPartListParts listParts(String volumeName,
     throw new UnsupportedOperationException("Ozone REST protocol does not " +
         "support this operation.");
   }
+
+  /**
+   * Get CanonicalServiceName for ozone delegation token.
+   * @return Canonical Service Name of ozone delegation token.
+   */
+  public String getCanonicalServiceName(){
+    throw new UnsupportedOperationException("Ozone REST protocol does not " +
+        "support this operation.");
+  }
 }

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -25,6 +25,7 @@
 import org.apache.hadoop.crypto.CryptoInputStream;
 import org.apache.hadoop.crypto.CryptoOutputStream;
 import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProviderTokenIssuer;
 import org.apache.hadoop.fs.FileEncryptionInfo;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.StorageType;
@@ -92,6 +93,7 @@
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
+import java.net.URI;
 import java.util.*;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
@@ -101,7 +103,7 @@
  * to execute client calls. This uses RPC protocol for communication
  * with the servers.
  */
-public class RpcClient implements ClientProtocol {
+public class RpcClient implements ClientProtocol, KeyProviderTokenIssuer {
 
   private static final Logger LOG =
       LoggerFactory.getLogger(RpcClient.class);
@@ -124,6 +126,7 @@ public class RpcClient implements ClientProtocol {
   private final long watchTimeout;
   private final ClientId clientId = ClientId.randomId();
   private final int maxRetryCount;
+  private Text dtService;
 
    /**
     * Creates RpcClient instance with the given configuration.
@@ -208,6 +211,8 @@ public RpcClient(Configuration conf) throws IOException {
     maxRetryCount =
         conf.getInt(OzoneConfigKeys.OZONE_CLIENT_MAX_RETRIES, OzoneConfigKeys.
             OZONE_CLIENT_MAX_RETRIES_DEFAULT);
+    dtService =
+        getOMProxyProvider().getProxy().getDelegationTokenService();
   }
 
   private InetSocketAddress getScmAddressForClient() throws IOException {
@@ -452,12 +457,11 @@ public Token<OzoneTokenIdentifier> getDelegationToken(Text renewer)
     Token<OzoneTokenIdentifier> token =
         ozoneManagerClient.getDelegationToken(renewer);
     if (token != null) {
-      Text dtService =
-          getOMProxyProvider().getProxy().getDelegationTokenService();
       token.setService(dtService);
-      LOG.debug("Created token {}", token);
+      LOG.debug("Created token {} for dtService {}", token, dtService);
     } else {
-      LOG.debug("Cannot get ozone delegation token from {}", renewer);
+      LOG.debug("Cannot get ozone delegation token for renewer {} to access " +
+          "service {}", renewer, dtService);
     }
     return token;
   }
@@ -646,10 +650,8 @@ private KeyProvider.KeyVersion getDEK(FileEncryptionInfo feInfo)
     // check crypto protocol version
     OzoneKMSUtil.checkCryptoProtocolVersion(feInfo);
     KeyProvider.KeyVersion decrypted;
-    // TODO: support get kms uri from om rpc server.
     decrypted = OzoneKMSUtil.decryptEncryptedDataEncryptionKey(feInfo,
-        OzoneKMSUtil.getKeyProvider(conf, OzoneKMSUtil.getKeyProviderUri(
-            ugi, null, null, conf)));
+        getKeyProvider());
     return decrypted;
   }
 
@@ -966,4 +968,25 @@ public OzoneMultipartUploadPartListParts listParts(String volumeName,
 
   }
 
+  @Override
+  public KeyProvider getKeyProvider() throws IOException {
+    return OzoneKMSUtil.getKeyProvider(conf, getKeyProviderUri());
+  }
+
+  @Override
+  public URI getKeyProviderUri() throws IOException {
+    // TODO: fix me to support kms instances for difference OMs
+    return OzoneKMSUtil.getKeyProviderUri(ugi,
+        null, null, conf);
+  }
+
+  @Override
+  public String getCanonicalServiceName() {
+    return (dtService != null) ? dtService.toString() : null;
+  }
+
+  @Override
+  public Token<?> getDelegationToken(String renewer) throws IOException {
+    return getDelegationToken(renewer == null ? null : new Text(renewer));
+  }
 }

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java

@@ -134,10 +134,8 @@ public Token<OzoneTokenIdentifier> createToken(Text owner, Text renewer,
     addToTokenStore(identifier, password, expiryTime);
     Token<OzoneTokenIdentifier> token = new Token<>(identifier.getBytes(),
         password, identifier.getKind(), getService());
-    if (LOG.isTraceEnabled()) {
-      String tokenId = identifier.toStringStable();
-      LOG.trace("Issued delegation token -> expiryTime:{},tokenId:{}",
-          expiryTime, tokenId);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Created delegation token: {}", token);
     }
     return token;
   }

hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env

@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HDDS_VERSION=${hdds.version}
+HADOOP_VERSION=3

hadoop-ozone/dist/src/main/compose/ozonesecure-mr/README.md

@@ -0,0 +1,76 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+# Secure Docker-compose with KMS, Yarn RM and NM
+This docker compose allows to test Sample Map Reduce Jobs with OzoneFileSystem
+It is a superset of ozonesecure docker-compose, which add Yarn NM/RM in addition
+to Ozone OM/SCM/NM/DN and Kerberos KDC.
+
+## Basic setup
+
+```
+cd hadoop-ozone/dist/target/ozone-0.5.0-SNAPSHOT/compose/ozonesecure-mr
+
+docker-compose up -d
+```
+
+## Ozone Manager Setup
+
+```
+docker-compose exec om bash
+
+kinit -kt /etc/security/keytabs/testuser.keytab testuser/[email protected]
+
+ozone sh volume create /vol1
+
+ozone sh bucket create /vol1/bucket1
+
+ozone sh key put /vol1/bucket1/key1 LICENSE.txt
+
+ozone fs -ls o3fs://bucket1.vol1/
+```
+
+## Yarn Resource Manager Setup
+```
+docker-compose exec rm bash
+
+kinit -kt /etc/security/keytabs/hadoop.keytab hadoop/[email protected]
+export HADOOP_MAPRED_HOME=/opt/hadoop/share/hadoop/mapreduce
+
+export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/opt/hadoop/share/hadoop/mapreduce/*:/opt/ozone/share/ozone/lib/hadoop-ozone-filesystem-lib-current-0.5.0-SNAPSHOT.jar
+
+hadoop fs -mkdir /user
+hadoop fs -mkdir /user/hadoop
+```
+
+## Run Examples
+
+### WordCount
+Status: Fully working with HDDS-1299
+```
+yarn jar $HADOOP_MAPRED_HOME/hadoop-mapreduce-examples-*.jar wordcount o3fs://bucket1.vol1/key1 o3fs://bucket1.vol1/key1.count
+
+hadoop fs -cat /key1.count/part-r-00000
+```
+
+### Pi
+Status: Not fully working yet, tracked by HDDS-1317
+```
+yarn jar $HADOOP_MAPRED_HOME/hadoop-mapreduce-examples-*.jar pi 10 100
+```
+
+### RandomWrite
+Status: Not fully working yet, tracked by HDDS-1317
+```
+yarn jar $HADOOP_MAPRED_HOME/hadoop-mapreduce-examples-*.jar randomwriter -Dtest.randomwrite.total_bytes=10000000  o3fs://bucket1.vol1/randomwrite.out
+```