From 1c28597e1d8d194de7ef5ab4a1dc3a356b2362ba Mon Sep 17 00:00:00 2001 From: sumangala Date: Thu, 26 Aug 2021 20:41:21 +0530 Subject: [PATCH 1/6] rest op fix --- .../oauth2/CustomTokenProviderAdapter.java | 6 ++++++ .../fs/azurebfs/services/AbfsClient.java | 5 +++++ .../azurebfs/AbstractAbfsIntegrationTest.java | 5 +++++ .../ITestAbfsRestOperationException.java | 9 +++++--- .../oauth2/RetryTestTokenProvider.java | 21 +++++++++++++------ .../fs/azurebfs/services/TestAbfsClient.java | 4 ++++ 6 files changed, 41 insertions(+), 9 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java index 14914101e5cca..6ae0962fbfd82 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.net.URI; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -138,4 +139,9 @@ public String getUserAgentSuffix() { String suffix = ExtensionHelper.getUserAgentSuffix(adaptee, ""); return suffix != null ? suffix : ""; } + + @VisibleForTesting + protected CustomTokenProviderAdaptee getCustomTokenProviderAdaptee() { + return adaptee; + } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 27206959ba533..50546c79beaf3 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -1204,4 +1204,9 @@ public ListenableFuture submit(Runnable runnable) { public void addCallback(ListenableFuture future, FutureCallback callback) { Futures.addCallback(future, callback, executorService); } + + @VisibleForTesting + protected AccessTokenProvider getTokenProvider() { + return tokenProvider; + } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java index d4e44c37f632a..c7baf94576ca0 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java @@ -37,10 +37,12 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azurebfs.constants.FSOperationType; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException; +import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider; import org.apache.hadoop.fs.azurebfs.security.AbfsDelegationTokenManager; import org.apache.hadoop.fs.azurebfs.services.AbfsClient; import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream; import org.apache.hadoop.fs.azurebfs.services.AuthType; +import org.apache.hadoop.fs.azurebfs.services.TestAbfsClient; import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore; import org.apache.hadoop.fs.azure.NativeAzureFileSystem; import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation; @@ -241,6 +243,9 @@ public Hashtable call() throws Exception { } } + public AccessTokenProvider getAccessTokenProvider(final AzureBlobFileSystem fs) { + return TestAbfsClient.getAccessTokenProvider(fs.getAbfsStore().getClient()); + } public void loadConfiguredFileSystem() throws Exception { // disable auto-creation of filesystem diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java index a71e7bc815f75..6ba9f58627870 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java @@ -111,7 +111,10 @@ public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exce final AzureBlobFileSystem fs1 = (AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(), config); - RetryTestTokenProvider.ResetStatusToFirstTokenFetch(); + RetryTestTokenProvider retryTestTokenProvider + = RetryTestTokenProvider.getCurrentRetryTestProviderInstance( + getAccessTokenProvider(fs1)); + retryTestTokenProvider.resetStatusToFirstTokenFetch(); intercept(Exception.class, ()-> { @@ -120,9 +123,9 @@ public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exce // Number of retries done should be as configured Assert.assertTrue( - "Number of token fetch retries (" + RetryTestTokenProvider.reTryCount + "Number of token fetch retries (" + retryTestTokenProvider.getReTryCount() + ") done, does not match with fs.azure.custom.token.fetch.retry.count configured (" + numOfRetries - + ")", RetryTestTokenProvider.reTryCount == numOfRetries); + + ")", retryTestTokenProvider.getReTryCount() == numOfRetries); } @Test diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java index 3566ebbaaaa2a..05b61a2d1f7f5 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java @@ -30,12 +30,12 @@ */ public class RetryTestTokenProvider implements CustomTokenProviderAdaptee { - // Need to track first token fetch otherwise will get counted as a retry too. - private static boolean isThisFirstTokenFetch = true; - public static int reTryCount = 0; + private static final Logger LOG = LoggerFactory.getLogger( + RetryTestTokenProvider.class); - private static final Logger LOG = LoggerFactory - .getLogger(RetryTestTokenProvider.class); + // Need to track first token fetch otherwise will get counted as a retry too. + private boolean isThisFirstTokenFetch = true; + public int reTryCount = 0; @Override public void initialize(Configuration configuration, String accountName) @@ -43,7 +43,7 @@ public void initialize(Configuration configuration, String accountName) } - public static void ResetStatusToFirstTokenFetch() { + public void resetStatusToFirstTokenFetch() { isThisFirstTokenFetch = true; reTryCount = 0; } @@ -64,4 +64,13 @@ public String getAccessToken() throws IOException { public Date getExpiryTime() { return new Date(); } + + public static RetryTestTokenProvider getCurrentRetryTestProviderInstance( + AccessTokenProvider customTokenProvider) { + return (RetryTestTokenProvider) ((CustomTokenProviderAdapter) customTokenProvider).getCustomTokenProviderAdaptee(); + } + + public int getReTryCount() { + return reTryCount; + } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java index a725bf3175a5c..0a1dca7e7d8d7 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java @@ -395,4 +395,8 @@ public static AbfsRestOperation getRestOp(AbfsRestOperationType type, url, requestHeaders); } + + public static AccessTokenProvider getAccessTokenProvider(AbfsClient client) { + return client.getTokenProvider(); + } } From db02e3bace1a940ce73361db06f10646ed2a9094 Mon Sep 17 00:00:00 2001 From: sumangala Date: Thu, 26 Aug 2021 20:53:42 +0530 Subject: [PATCH 2/6] pom - exclude streamstats test from parallel --- hadoop-tools/hadoop-azure/pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml index cc773ab777fc3..25ad85b363731 100644 --- a/hadoop-tools/hadoop-azure/pom.xml +++ b/hadoop-tools/hadoop-azure/pom.xml @@ -506,6 +506,9 @@ **/azurebfs/Test*.java **/azurebfs/**/Test*.java + + **/azurebfs/ITestAbfsStreamStatistics*.java + @@ -555,6 +558,7 @@ **/azurebfs/ITestAzureBlobFileSystemListStatus.java **/azurebfs/extensions/ITestAbfsDelegationTokens.java **/azurebfs/ITestSmallWriteOptimization.java + **/azurebfs/ITestAbfsStreamStatistics*.java @@ -595,6 +599,9 @@ **/azurebfs/ITestAzureBlobFileSystemListStatus.java **/azurebfs/extensions/ITestAbfsDelegationTokens.java **/azurebfs/ITestSmallWriteOptimization.java + + **/azurebfs/ITestAbfsStreamStatistics*.java + From 407121672c4167f75588e3522713ee7ae2243b80 Mon Sep 17 00:00:00 2001 From: sumangala Date: Thu, 26 Aug 2021 22:19:27 +0530 Subject: [PATCH 3/6] checkstyle --- .../hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java index 05b61a2d1f7f5..92f8ad0bb84a4 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java @@ -35,7 +35,7 @@ public class RetryTestTokenProvider implements CustomTokenProviderAdaptee { // Need to track first token fetch otherwise will get counted as a retry too. private boolean isThisFirstTokenFetch = true; - public int reTryCount = 0; + private int reTryCount = 0; @Override public void initialize(Configuration configuration, String accountName) From dc2f22e8cbc70d762148e5d4f87ce2c630100ac4 Mon Sep 17 00:00:00 2001 From: sumangala Date: Sun, 29 Aug 2021 17:41:53 +0530 Subject: [PATCH 4/6] restop test review comments --- .../azurebfs/ITestAbfsRestOperationException.java | 8 ++++---- .../fs/azurebfs/oauth2/RetryTestTokenProvider.java | 14 +++++++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java index 6ba9f58627870..b1705cad01568 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java @@ -122,10 +122,10 @@ public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exce }); // Number of retries done should be as configured - Assert.assertTrue( - "Number of token fetch retries (" + retryTestTokenProvider.getReTryCount() - + ") done, does not match with fs.azure.custom.token.fetch.retry.count configured (" + numOfRetries - + ")", retryTestTokenProvider.getReTryCount() == numOfRetries); + Assert.assertEquals( + "Number of token fetch retries done does not match with fs.azure" + + ".custom.token.fetch.retry.count configured", numOfRetries, + retryTestTokenProvider.getRetryCount()); } @Test diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java index 92f8ad0bb84a4..7f828060a346c 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java @@ -35,7 +35,7 @@ public class RetryTestTokenProvider implements CustomTokenProviderAdaptee { // Need to track first token fetch otherwise will get counted as a retry too. private boolean isThisFirstTokenFetch = true; - private int reTryCount = 0; + private int retryCount = 0; @Override public void initialize(Configuration configuration, String accountName) @@ -43,9 +43,13 @@ public void initialize(Configuration configuration, String accountName) } + /** + * Clear earlier retry details and reset RetryTestTokenProvider instance to + * state of first access token fetch call + */ public void resetStatusToFirstTokenFetch() { isThisFirstTokenFetch = true; - reTryCount = 0; + retryCount = 0; } @Override @@ -53,7 +57,7 @@ public String getAccessToken() throws IOException { if (isThisFirstTokenFetch) { isThisFirstTokenFetch = false; } else { - reTryCount++; + retryCount++; } LOG.debug("RetryTestTokenProvider: Throw an exception in fetching tokens"); @@ -70,7 +74,7 @@ public static RetryTestTokenProvider getCurrentRetryTestProviderInstance( return (RetryTestTokenProvider) ((CustomTokenProviderAdapter) customTokenProvider).getCustomTokenProviderAdaptee(); } - public int getReTryCount() { - return reTryCount; + public int getRetryCount() { + return retryCount; } } From c78d1839322f664d1f1069db69ec0ec0b98d603d Mon Sep 17 00:00:00 2001 From: sumangala Date: Tue, 14 Sep 2021 23:35:10 +0530 Subject: [PATCH 5/6] pom run test --- hadoop-tools/hadoop-azure/pom.xml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml index 7de5b3b3bf368..075716e91e64c 100644 --- a/hadoop-tools/hadoop-azure/pom.xml +++ b/hadoop-tools/hadoop-azure/pom.xml @@ -506,9 +506,6 @@ **/azurebfs/Test*.java **/azurebfs/**/Test*.java - - **/azurebfs/ITestAbfsStreamStatistics*.java - @@ -601,9 +598,7 @@ **/azurebfs/extensions/ITestAbfsDelegationTokens.java **/azurebfs/ITestSmallWriteOptimization.java **/azurebfs/services/ITestReadBufferManager.java - - **/azurebfs/ITestAbfsStreamStatistics*.java - + **/azurebfs/ITestAbfsStreamStatistics*.java From 5f5b3325fbbdfd33a6f058c1a521122a158b8d6b Mon Sep 17 00:00:00 2001 From: sumangala Date: Thu, 16 Sep 2021 19:26:07 +0530 Subject: [PATCH 6/6] add . javadoc --- .../hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java index 7f828060a346c..7427add29086c 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java @@ -45,7 +45,7 @@ public void initialize(Configuration configuration, String accountName) /** * Clear earlier retry details and reset RetryTestTokenProvider instance to - * state of first access token fetch call + * state of first access token fetch call. */ public void resetStatusToFirstTokenFetch() { isThisFirstTokenFetch = true;