From 889f223d98a353f3a2784ea44e7faf1840186aea Mon Sep 17 00:00:00 2001 From: Xiaoyu Yao Date: Sat, 12 Sep 2020 18:34:58 -0700 Subject: [PATCH 1/6] HDDS-4241. Support HADOOP_TOKEN_FILE_LOCATION for Ozone token CLI. --- .../ozone/security/OzoneTokenIdentifier.java | 5 ++- .../security/ozone-secure-token.robot | 18 +++++++-- .../ozone/shell/token/GetTokenHandler.java | 8 +++- .../ozone/shell/token/PrintTokenHandler.java | 3 +- .../ozone/shell/token/RenewTokenHandler.java | 4 +- .../hadoop/ozone/shell/token/TokenOption.java | 38 +++++++++++++++---- 6 files changed, 59 insertions(+), 17 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java index c0b1ddbd1dd9..54eb7b940598 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java @@ -21,6 +21,7 @@ import java.io.DataInputStream; import java.io.DataOutput; import java.io.IOException; +import java.time.Instant; import java.util.Arrays; import org.apache.commons.lang3.builder.EqualsBuilder; @@ -374,8 +375,8 @@ public String toString() { .append(" owner=").append(getOwner()) .append(", renewer=").append(getRenewer()) .append(", realUser=").append(getRealUser()) - .append(", issueDate=").append(getIssueDate()) - .append(", maxDate=").append(getMaxDate()) + .append(", issueDate=").append(Instant.ofEpochMilli(getIssueDate())) + .append(", maxDate=").append(Instant.ofEpochMilli(getMaxDate())) .append(", sequenceNumber=").append(getSequenceNumber()) .append(", masterKeyId=").append(getMasterKeyId()) .append(", strToSign=").append(getStrToSign()) diff --git a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot index 5823143638ba..5269753caf5c 100644 --- a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot +++ b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot @@ -22,9 +22,19 @@ Resource ../commonlib.robot Test Timeout 5 minutes *** Keywords *** -Get Token in Secure Cluster - Execute ozone sh token get > /tmp/token.txt - File Should Not Be Empty /tmp/token.txt +Get and use Token in Secure Cluster + Execute ozone sh token get > /tmp/ozone.token + File Should Not Be Empty /tmp/ozone.token + Execute kdestroy + ${output} = klist + Should contain ${output} No credentials cache found + Execute export HADOOP_TOKEN_FILE_LOCATION=/tmp/ozone.token + ${output} = Execute ozone sh volume list / + Should not contain ${output} Client cannot authenticate + Execute unset HADOOP_TOKEN_FILE_LOCATION + ${output} = Execute ozone sh volume list / + Should contain ${output} Client cannot authenticate + Execute Kinit test user testuser testuser.keytab Get Token in Unsecure Cluster ${output} = Execute ozone sh token get @@ -59,7 +69,7 @@ Cancel Token in Unsecure Cluster Should Contain ${output} only when security is enabled Token Test in Secure Cluster - Get Token in Secure Cluster + Get and use Token in Secure Cluster Print Valid Token File Renew Token in Secure Cluster Cancel Token in Secure Cluster diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/GetTokenHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/GetTokenHandler.java index 5bf8ccb83eb0..d380b657b663 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/GetTokenHandler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/GetTokenHandler.java @@ -46,6 +46,9 @@ public class GetTokenHandler extends Handler { @CommandLine.Mixin private RenewerOption renewer; + @CommandLine.Mixin + private TokenOption tokenFile; + @Override protected OzoneAddress getAddress() throws OzoneClientException { return new OzoneAddress(uri); @@ -66,7 +69,10 @@ protected void execute(OzoneClient client, OzoneAddress address) err().println("Error: Get delegation token operation failed. " + "Check OzoneManager logs for more details."); } else { - printObjectAsJson(token.encodeToUrlString()); + System.out.println("Successfully get token for service " + + token.getService()); + System.out.println(token.toString()); + tokenFile.persistToken(token); } } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/PrintTokenHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/PrintTokenHandler.java index 632a9d277487..4e748f6a69c1 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/PrintTokenHandler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/PrintTokenHandler.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.shell.token; -import org.apache.hadoop.hdds.server.JsonUtils; import org.apache.hadoop.ozone.security.OzoneTokenIdentifier; import org.apache.hadoop.security.token.Token; @@ -42,7 +41,7 @@ public class PrintTokenHandler implements Callable { public Void call() throws Exception { if (tokenFile.exists()) { Token token = tokenFile.decode(); - System.out.print(JsonUtils.toJsonStringWithDefaultPrettyPrinter(token)); + System.out.print(token.toString()); } return null; } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/RenewTokenHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/RenewTokenHandler.java index 6b5e7a1e5129..581093050ae0 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/RenewTokenHandler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/RenewTokenHandler.java @@ -24,6 +24,7 @@ import picocli.CommandLine.Command; import java.io.IOException; +import java.time.Instant; /** * Executes renewDelegationToken api. @@ -36,6 +37,7 @@ public class RenewTokenHandler extends TokenHandler { protected void execute(OzoneClient client, OzoneAddress address) throws IOException, OzoneClientException { long expiryTime = client.getObjectStore().renewDelegationToken(getToken()); - out().printf("Token renewed successfully, expiry time: %s.%n", expiryTime); + out().printf("Token renewed successfully, expiry time: %s.%n", + Instant.ofEpochMilli(expiryTime)); } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/TokenOption.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/TokenOption.java index 61d479bb36e8..6d7e857d689d 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/TokenOption.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/TokenOption.java @@ -18,13 +18,17 @@ package org.apache.hadoop.ozone.shell.token; import org.apache.hadoop.ozone.security.OzoneTokenIdentifier; +import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; import picocli.CommandLine; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; /** * Option for token file. @@ -33,7 +37,7 @@ public class TokenOption { @CommandLine.Option(names = {"--token", "-t"}, description = "file containing encoded token", - defaultValue = "/tmp/token.txt", + defaultValue = "/tmp/ozone.token", showDefaultValue = CommandLine.Help.Visibility.ALWAYS) private File tokenFile; @@ -47,10 +51,30 @@ public boolean exists() { } public Token decode() throws IOException { - Token token = new Token<>(); - token.decodeFromUrlString(new String(Files.readAllBytes(tokenFile.toPath()), - StandardCharsets.UTF_8)); - return token; + Credentials creds = new Credentials(); + try (FileInputStream fis = new FileInputStream(tokenFile)) { + try (DataInputStream dis = new DataInputStream(fis)) { + creds.readTokenStorageStream(dis); + } + } + for (Token token: creds.getAllTokens()) { + if (token.getKind().equals(OzoneTokenIdentifier.KIND_NAME)) { + return (Token) token; + } + } + return null; } + public void persistToken(Token token) + throws IOException { + try (FileOutputStream fos = new FileOutputStream(tokenFile)) { + try (DataOutputStream dos = new DataOutputStream(fos)) { + Credentials ts = new Credentials(); + ts.addToken(token.getService(), token); + ts.writeTokenStorageToStream(dos); + System.out.println("Token persisted to " + tokenFile.toString() + + " successfully!"); + } + } + } } \ No newline at end of file From 49b88eb453fff97cdc5239eaff4841e80eedb667 Mon Sep 17 00:00:00 2001 From: Xiaoyu Yao Date: Mon, 14 Sep 2020 13:30:30 -0700 Subject: [PATCH 2/6] fix klist issue --- .../dist/src/main/smoketest/security/ozone-secure-token.robot | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot index 5269753caf5c..1362a9ab6d94 100644 --- a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot +++ b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot @@ -26,8 +26,6 @@ Get and use Token in Secure Cluster Execute ozone sh token get > /tmp/ozone.token File Should Not Be Empty /tmp/ozone.token Execute kdestroy - ${output} = klist - Should contain ${output} No credentials cache found Execute export HADOOP_TOKEN_FILE_LOCATION=/tmp/ozone.token ${output} = Execute ozone sh volume list / Should not contain ${output} Client cannot authenticate From abf9c6acb0ffb108ea172592397d3d1e9281d00d Mon Sep 17 00:00:00 2001 From: Xiaoyu Yao Date: Mon, 14 Sep 2020 14:48:04 -0700 Subject: [PATCH 3/6] fix acceptance test --- .../dist/src/main/smoketest/security/ozone-secure-token.robot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot index 1362a9ab6d94..b546c3dcf864 100644 --- a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot +++ b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot @@ -23,7 +23,7 @@ Test Timeout 5 minutes *** Keywords *** Get and use Token in Secure Cluster - Execute ozone sh token get > /tmp/ozone.token + Execute ozone sh token get -t /tmp/ozone.token File Should Not Be Empty /tmp/ozone.token Execute kdestroy Execute export HADOOP_TOKEN_FILE_LOCATION=/tmp/ozone.token From e4824b87a7a8bd9419c187acb2f3b0743d737077 Mon Sep 17 00:00:00 2001 From: Xiaoyu Yao Date: Tue, 15 Sep 2020 00:24:24 -0700 Subject: [PATCH 4/6] fix formatting --- .../org/apache/hadoop/ozone/shell/token/GetTokenHandler.java | 4 ++-- .../apache/hadoop/ozone/shell/token/PrintTokenHandler.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/GetTokenHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/GetTokenHandler.java index d380b657b663..d159d2233f95 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/GetTokenHandler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/GetTokenHandler.java @@ -69,9 +69,9 @@ protected void execute(OzoneClient client, OzoneAddress address) err().println("Error: Get delegation token operation failed. " + "Check OzoneManager logs for more details."); } else { - System.out.println("Successfully get token for service " + + out().println("Successfully get token for service " + token.getService()); - System.out.println(token.toString()); + out().println(token.toString()); tokenFile.persistToken(token); } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/PrintTokenHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/PrintTokenHandler.java index 4e748f6a69c1..ab02b76ad56a 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/PrintTokenHandler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/PrintTokenHandler.java @@ -41,7 +41,7 @@ public class PrintTokenHandler implements Callable { public Void call() throws Exception { if (tokenFile.exists()) { Token token = tokenFile.decode(); - System.out.print(token.toString()); + System.out.println(token.toString()); } return null; } From 5e3297b39cc8c5788b20607c8d0274668bf94e42 Mon Sep 17 00:00:00 2001 From: Xiaoyu Yao Date: Tue, 15 Sep 2020 13:26:31 -0700 Subject: [PATCH 5/6] acceptance update --- .../dist/src/main/smoketest/security/ozone-secure-token.robot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot index b546c3dcf864..51389c8fdc1d 100644 --- a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot +++ b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot @@ -32,7 +32,7 @@ Get and use Token in Secure Cluster Execute unset HADOOP_TOKEN_FILE_LOCATION ${output} = Execute ozone sh volume list / Should contain ${output} Client cannot authenticate - Execute Kinit test user testuser testuser.keytab + Run Keyword Kinit test user testuser testuser.keytab Get Token in Unsecure Cluster ${output} = Execute ozone sh token get From 0b16dd8efd8a9208704b60ea2f83083013217f63 Mon Sep 17 00:00:00 2001 From: Xiaoyu Yao Date: Tue, 15 Sep 2020 18:05:25 -0700 Subject: [PATCH 6/6] Set/Delete environment variable using robottest library --- .../src/main/smoketest/security/ozone-secure-token.robot | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot index 51389c8fdc1d..a84d2efbcabe 100644 --- a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot +++ b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot @@ -26,11 +26,11 @@ Get and use Token in Secure Cluster Execute ozone sh token get -t /tmp/ozone.token File Should Not Be Empty /tmp/ozone.token Execute kdestroy - Execute export HADOOP_TOKEN_FILE_LOCATION=/tmp/ozone.token + Set Environment Variable HADOOP_TOKEN_FILE_LOCATION /tmp/ozone.token ${output} = Execute ozone sh volume list / Should not contain ${output} Client cannot authenticate - Execute unset HADOOP_TOKEN_FILE_LOCATION - ${output} = Execute ozone sh volume list / + Remove Environment Variable HADOOP_TOKEN_FILE_LOCATION + ${output} = Execute and Ignore Error ozone sh volume list / Should contain ${output} Client cannot authenticate Run Keyword Kinit test user testuser testuser.keytab