From 74cd5dcb23c64151edd5bed549c32b94a5691abb Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Sun, 22 Sep 2024 23:31:48 -0700 Subject: [PATCH 1/6] Making is HNS enabled account specific setting --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 4 +- .../fs/azurebfs/AzureBlobFileSystemStore.java | 2 +- .../fs/azurebfs/ITestGetNameSpaceEnabled.java | 43 +++++++++++++++++++ 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 1892b13d372f6..cd54c0933b018 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -451,7 +451,9 @@ public AbfsConfiguration(final Configuration rawConfig, String accountName) } public Trilean getIsNamespaceEnabledAccount() { - return Trilean.getTrilean(isNamespaceEnabledAccount); + String isNamespaceEnabledAccountString + = getString(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, isNamespaceEnabledAccount); + return Trilean.getTrilean(isNamespaceEnabledAccountString); } /** diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index e903421f469a7..510ed733e2da1 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -405,7 +405,7 @@ private synchronized boolean getNamespaceEnabledInformationFromServer( } try { LOG.debug("Get root ACL status"); - getClient().getAclStatus(AbfsHttpConstants.ROOT_PATH, tracingContext); + getClient(AbfsServiceType.DFS).getAclStatus(AbfsHttpConstants.ROOT_PATH, tracingContext); // If getAcl succeeds, namespace is enabled. isNamespaceEnabled = Trilean.getTrilean(true); } catch (AbfsRestOperationException ex) { diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java index d4c58c9705a68..b1375bf123924 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java @@ -41,6 +41,7 @@ import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.net.HttpURLConnection.HTTP_UNAVAILABLE; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.accountProperty; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doReturn; @@ -126,6 +127,8 @@ private AzureBlobFileSystem getNewFSWithHnsConf( Configuration rawConfig = new Configuration(); rawConfig.addResource(TEST_CONFIGURATION_FILE_NAME); rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, isNamespaceEnabledAccount); + rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, + this.getAccountName()), isNamespaceEnabledAccount); rawConfig .setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, true); rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, @@ -271,4 +274,44 @@ private void ensureGetAclDetermineHnsStatusAccuratelyInternal(int statusCode, Mockito.verify(mockClient, times(1)) .getAclStatus(anyString(), any(TracingContext.class)); } + + @Test + public void testAccountSpecificConfig() throws Exception { + Configuration rawConfig = new Configuration(); + rawConfig.addResource(TEST_CONFIGURATION_FILE_NAME); + rawConfig.unset(FS_AZURE_ACCOUNT_IS_HNS_ENABLED); + rawConfig.unset(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, + this.getAccountName())); + String accountName1 = "account1.dfs.core.windows.net"; + String accountName2 = "account2.dfs.core.windows.net"; + String accountName3 = "account3.dfs.core.windows.net"; + String defaultUri1 = this.getTestUrl().replace(this.getAccountName(), accountName1); + String defaultUri2 = this.getTestUrl().replace(this.getAccountName(), accountName2); + String defaultUri3 = this.getTestUrl().replace(this.getAccountName(), accountName3); + + // Set account specific config for account 1 + rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, accountName1), TRUE_STR); + rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri1); + AzureBlobFileSystem fs1 = (AzureBlobFileSystem) FileSystem.newInstance(rawConfig); + + // Set account specific config for account 2 + rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, accountName2), FALSE_STR); + rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri2); + AzureBlobFileSystem fs2 = (AzureBlobFileSystem) FileSystem.newInstance(rawConfig); + + // Set account agnostic config for account 3 + rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, FALSE_STR); + rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri3); + AzureBlobFileSystem fs3 = (AzureBlobFileSystem) FileSystem.newInstance(rawConfig); + + Assertions.assertThat(getIsNamespaceEnabled(fs1)).describedAs( + "getIsNamespaceEnabled should return true when the " + + "account specific config is set as true").isTrue(); + Assertions.assertThat(getIsNamespaceEnabled(fs2)).describedAs( + "getIsNamespaceEnabled should return true when the " + + "account specific config is set as true").isFalse(); + Assertions.assertThat(getIsNamespaceEnabled(fs3)).describedAs( + "getIsNamespaceEnabled should return true when the " + + "account specific config is not set").isFalse(); + } } From ba947aaab557ae8c00696a0e4c2951789970ac19 Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Mon, 23 Sep 2024 01:26:53 -0700 Subject: [PATCH 2/6] Fixing Failing tests --- .../fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java | 3 ++- .../apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java index 44b1685a3f3f2..69d4f79f8b099 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java @@ -28,6 +28,7 @@ import org.junit.Test; import org.mockito.Mockito; +import org.apache.hadoop.fs.azurebfs.constants.AbfsServiceType; import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException; @@ -76,7 +77,7 @@ public void testGetAclCallOnHnsConfigAbsence() throws Exception { getRawConfiguration())); AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore()); AbfsClient client = Mockito.spy(fs.getAbfsClient()); - Mockito.doReturn(client).when(store).getClient(); + Mockito.doReturn(client).when(store).getClient(AbfsServiceType.DFS); Mockito.doThrow(TrileanConversionException.class) .when(store) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java index b1375bf123924..600a35a1e35ea 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.azurebfs.constants.AbfsServiceType; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; import org.apache.hadoop.fs.azurebfs.services.AbfsClient; import org.apache.hadoop.fs.azurebfs.services.AbfsDfsClient; @@ -250,7 +251,7 @@ private void ensureGetAclDetermineHnsStatusAccuratelyInternal(int statusCode, AzureBlobFileSystemStore store = Mockito.spy(getFileSystem().getAbfsStore()); AbfsClient mockClient = mock(AbfsClient.class); store.setNamespaceEnabled(Trilean.UNKNOWN); - doReturn(mockClient).when(store).getClient(); + doReturn(mockClient).when(store).getClient(AbfsServiceType.DFS); AbfsRestOperationException ex = new AbfsRestOperationException( statusCode, null, Integer.toString(statusCode), null); doThrow(ex).when(mockClient).getAclStatus(anyString(), any(TracingContext.class)); From ad926a39b5aa4910ab5e68e4bb2ba38b5e9584dd Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Mon, 23 Sep 2024 03:29:39 -0700 Subject: [PATCH 3/6] Updating Tests --- .../azurebfs/ITestAzureBlobFileSystemE2E.java | 4 +++ .../fs/azurebfs/ITestGetNameSpaceEnabled.java | 27 +++++++++++-------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java index c7a1fa91a98f9..00e9fc5b6143a 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java @@ -40,6 +40,7 @@ import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_HTTP_READ_TIMEOUT; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MAX_IO_RETRIES; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_TOLERATE_CONCURRENT_APPEND; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_HNS_ENABLED; import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathDoesNotExist; import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathExists; import static org.apache.hadoop.test.LambdaTestUtils.intercept; @@ -259,6 +260,9 @@ public void testHttpReadTimeout() throws Exception { public void testHttpTimeouts(int connectionTimeoutMs, int readTimeoutMs) throws Exception { + // This is to make sure File System creation goes through before network calls start failing. + assumeValidTestConfigPresent(this.getRawConfiguration(), FS_AZURE_ACCOUNT_IS_HNS_ENABLED); + Configuration conf = this.getRawConfiguration(); // set to small values that will cause timeouts conf.setInt(AZURE_HTTP_CONNECTION_TIMEOUT, connectionTimeoutMs); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java index 600a35a1e35ea..62d5f3b841ef4 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java @@ -290,27 +290,32 @@ public void testAccountSpecificConfig() throws Exception { String defaultUri2 = this.getTestUrl().replace(this.getAccountName(), accountName2); String defaultUri3 = this.getTestUrl().replace(this.getAccountName(), accountName3); - // Set account specific config for account 1 - rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, accountName1), TRUE_STR); + // Set both account specific and account agnostic config for account 1 + rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, accountName1), FALSE_STR); + rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, TRUE_STR); rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri1); AzureBlobFileSystem fs1 = (AzureBlobFileSystem) FileSystem.newInstance(rawConfig); + // Assert that account specific config takes precedence + Assertions.assertThat(getIsNamespaceEnabled(fs1)).describedAs( + "getIsNamespaceEnabled should return true when the " + + "account specific config is set as true").isFalse(); - // Set account specific config for account 2 + // Set only the account specific config for account 2 rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, accountName2), FALSE_STR); + rawConfig.unset(FS_AZURE_ACCOUNT_IS_HNS_ENABLED); rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri2); AzureBlobFileSystem fs2 = (AzureBlobFileSystem) FileSystem.newInstance(rawConfig); + // Assert that account specific config is enough. + Assertions.assertThat(getIsNamespaceEnabled(fs2)).describedAs( + "getIsNamespaceEnabled should return true when the " + + "account specific config is set as true").isFalse(); - // Set account agnostic config for account 3 + // Set only account agnostic config for account 3 rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, FALSE_STR); + rawConfig.unset(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, accountName3)); rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri3); AzureBlobFileSystem fs3 = (AzureBlobFileSystem) FileSystem.newInstance(rawConfig); - - Assertions.assertThat(getIsNamespaceEnabled(fs1)).describedAs( - "getIsNamespaceEnabled should return true when the " - + "account specific config is set as true").isTrue(); - Assertions.assertThat(getIsNamespaceEnabled(fs2)).describedAs( - "getIsNamespaceEnabled should return true when the " - + "account specific config is set as true").isFalse(); + // Assert that account agnostic config is enough. Assertions.assertThat(getIsNamespaceEnabled(fs3)).describedAs( "getIsNamespaceEnabled should return true when the " + "account specific config is not set").isFalse(); From 52f928d751b534ad015c549ea9d765eeab2ee9ab Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Wed, 25 Sep 2024 22:14:08 -0700 Subject: [PATCH 4/6] Resolving Comments --- .../org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java | 7 +++++++ .../hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java | 3 +++ 2 files changed, 10 insertions(+) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index cd54c0933b018..8c1524a660212 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -450,6 +450,13 @@ public AbfsConfiguration(final Configuration rawConfig, String accountName) this(rawConfig, accountName, AbfsServiceType.DFS); } + /** + * Returns the account type as per the user configuration. Gets the account + * specific value if it exists, then looks for an account agnostic value. + * If not configured driver makes additional getAcl call to determine + * the account type during file system initialization. + * @return TRUE/FALSE value if configured, UNKNOWN if not configured. + */ public Trilean getIsNamespaceEnabledAccount() { String isNamespaceEnabledAccountString = getString(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, isNamespaceEnabledAccount); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java index 62d5f3b841ef4..c0965b47ec83f 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java @@ -319,5 +319,8 @@ public void testAccountSpecificConfig() throws Exception { Assertions.assertThat(getIsNamespaceEnabled(fs3)).describedAs( "getIsNamespaceEnabled should return true when the " + "account specific config is not set").isFalse(); + fs1.close(); + fs2.close(); + fs3.close(); } } From cc45f99056e08952fd10aba12461eb3ac472364c Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Thu, 26 Sep 2024 04:00:14 -0700 Subject: [PATCH 5/6] Resolving Comments --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 7 +- .../fs/azurebfs/ITestGetNameSpaceEnabled.java | 83 +++++++++++-------- 2 files changed, 52 insertions(+), 38 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 8c1524a660212..db9425b022107 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -451,16 +451,15 @@ public AbfsConfiguration(final Configuration rawConfig, String accountName) } /** - * Returns the account type as per the user configuration. Gets the account + * Returns the account type as per the user configuration. Gets the account * specific value if it exists, then looks for an account agnostic value. * If not configured driver makes additional getAcl call to determine * the account type during file system initialization. * @return TRUE/FALSE value if configured, UNKNOWN if not configured. */ public Trilean getIsNamespaceEnabledAccount() { - String isNamespaceEnabledAccountString - = getString(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, isNamespaceEnabledAccount); - return Trilean.getTrilean(isNamespaceEnabledAccountString); + return Trilean.getTrilean( + getString(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, isNamespaceEnabledAccount)); } /** diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java index c0965b47ec83f..6a0d3f8c08168 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java @@ -30,6 +30,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.azurebfs.constants.AbfsServiceType; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException; import org.apache.hadoop.fs.azurebfs.services.AbfsClient; import org.apache.hadoop.fs.azurebfs.services.AbfsDfsClient; import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation; @@ -42,6 +43,7 @@ import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.net.HttpURLConnection.HTTP_UNAVAILABLE; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MAX_IO_RETRIES; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.accountProperty; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; @@ -283,44 +285,57 @@ public void testAccountSpecificConfig() throws Exception { rawConfig.unset(FS_AZURE_ACCOUNT_IS_HNS_ENABLED); rawConfig.unset(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getAccountName())); - String accountName1 = "account1.dfs.core.windows.net"; - String accountName2 = "account2.dfs.core.windows.net"; - String accountName3 = "account3.dfs.core.windows.net"; - String defaultUri1 = this.getTestUrl().replace(this.getAccountName(), accountName1); - String defaultUri2 = this.getTestUrl().replace(this.getAccountName(), accountName2); - String defaultUri3 = this.getTestUrl().replace(this.getAccountName(), accountName3); - - // Set both account specific and account agnostic config for account 1 - rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, accountName1), FALSE_STR); + String testAccountName = "testAccount.dfs.core.windows.net"; + String otherAccountName = "otherAccount.dfs.core.windows.net"; + String defaultUri = this.getTestUrl().replace(this.getAccountName(), testAccountName); + String otherUri = this.getTestUrl().replace(this.getAccountName(), otherAccountName); + + // Set both account specific and account agnostic config for test account + rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName), FALSE_STR); rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, TRUE_STR); - rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri1); - AzureBlobFileSystem fs1 = (AzureBlobFileSystem) FileSystem.newInstance(rawConfig); // Assert that account specific config takes precedence - Assertions.assertThat(getIsNamespaceEnabled(fs1)).describedAs( - "getIsNamespaceEnabled should return true when the " - + "account specific config is set as true").isFalse(); - - // Set only the account specific config for account 2 - rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, accountName2), FALSE_STR); + rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri); + assertFileSystemInitWithExpectedHNSSettings(rawConfig, false); + // Assert that other account still uses account agnostic config + rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, otherUri); + assertFileSystemInitWithExpectedHNSSettings(rawConfig, true); + + // Set only the account specific config for test account + rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName), FALSE_STR); rawConfig.unset(FS_AZURE_ACCOUNT_IS_HNS_ENABLED); - rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri2); - AzureBlobFileSystem fs2 = (AzureBlobFileSystem) FileSystem.newInstance(rawConfig); - // Assert that account specific config is enough. - Assertions.assertThat(getIsNamespaceEnabled(fs2)).describedAs( - "getIsNamespaceEnabled should return true when the " - + "account specific config is set as true").isFalse(); + // Assert that only account specific config is enough for test account + rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri); + assertFileSystemInitWithExpectedHNSSettings(rawConfig, false); - // Set only account agnostic config for account 3 + // Set only account agnostic config rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, FALSE_STR); - rawConfig.unset(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, accountName3)); - rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri3); - AzureBlobFileSystem fs3 = (AzureBlobFileSystem) FileSystem.newInstance(rawConfig); - // Assert that account agnostic config is enough. - Assertions.assertThat(getIsNamespaceEnabled(fs3)).describedAs( - "getIsNamespaceEnabled should return true when the " - + "account specific config is not set").isFalse(); - fs1.close(); - fs2.close(); - fs3.close(); + rawConfig.unset(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName)); + rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri); + assertFileSystemInitWithExpectedHNSSettings(rawConfig, false); + + // Unset both account specific and account agnostic config + rawConfig.unset(FS_AZURE_ACCOUNT_IS_HNS_ENABLED); + rawConfig.unset(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName)); + rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri); + rawConfig.set(AZURE_MAX_IO_RETRIES, "0"); + // Assert that file system init fails with UnknownHost exception as getAcl() is needed. + try { + assertFileSystemInitWithExpectedHNSSettings(rawConfig, false); + } catch (Exception e) { + Assertions.assertThat(e.getCause().getMessage()) + .describedAs("getAcl() to determine HNS Nature of account should" + + "fail with Unknown Host Exception").contains("UnknownHostException"); + } + } + + private void assertFileSystemInitWithExpectedHNSSettings( + Configuration configuration, boolean expectedIsHnsEnabledValue) throws IOException { + try (AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.newInstance(configuration)) { + Assertions.assertThat(getIsNamespaceEnabled(fs)).describedAs( + "getIsNamespaceEnabled should return true when the " + + "account specific config is not set").isEqualTo(expectedIsHnsEnabledValue); + } catch (Exception e) { + throw e; + } } } From f3d371f7cf916e362e8b370a3754b77000af5f22 Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Thu, 26 Sep 2024 05:28:32 -0700 Subject: [PATCH 6/6] Yetus Checks --- .../java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java | 2 +- .../org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index db9425b022107..eb12827ca7bd7 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -451,7 +451,7 @@ public AbfsConfiguration(final Configuration rawConfig, String accountName) } /** - * Returns the account type as per the user configuration. Gets the account + * Returns the account type as per the user configuration. Gets the account * specific value if it exists, then looks for an account agnostic value. * If not configured driver makes additional getAcl call to determine * the account type during file system initialization. diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java index 6a0d3f8c08168..5732ce8cfea70 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java @@ -30,7 +30,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.azurebfs.constants.AbfsServiceType; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; -import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException; import org.apache.hadoop.fs.azurebfs.services.AbfsClient; import org.apache.hadoop.fs.azurebfs.services.AbfsDfsClient; import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation;