From 5af9306030571b52305f789a11aff2e5517ed0dd Mon Sep 17 00:00:00 2001 From: Sneha Vijayarajan Date: Tue, 3 Sep 2019 01:54:27 +0530 Subject: [PATCH 1/3] Enable config control for openSSL socket connections for ADLS Gen1 --- .../src/main/resources/core-default.xml | 11 ++++ hadoop-tools/hadoop-azure-datalake/pom.xml | 2 +- .../org/apache/hadoop/fs/adl/AdlConfKeys.java | 1 + .../apache/hadoop/fs/adl/AdlFileSystem.java | 4 ++ .../src/site/markdown/troubleshooting_adl.md | 10 +++ .../fs/adl/live/TestAdlSdkConfiguration.java | 66 ++++++++++++++++++- 6 files changed, 92 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 7ed3e8a32dac6..920d4c9ce7971 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -3373,6 +3373,17 @@ + + adl.ssl.channel.mode + + + When OpenSSL - SSL socket connections are created in OpenSSL mode. + When Default_JSE - SSL socket connections are created in the default JSE mode. + When Default (default) - SSL socket connections are attempted with OpenSSL + and will fallback to Default_JSE mode if OpenSSL is not available at runtime. + + + diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml b/hadoop-tools/hadoop-azure-datalake/pom.xml index 6e73eaacc174c..9952861a2f97d 100644 --- a/hadoop-tools/hadoop-azure-datalake/pom.xml +++ b/hadoop-tools/hadoop-azure-datalake/pom.xml @@ -33,7 +33,7 @@ 0.9.1 UTF-8 true - 2.3.3 + 2.3.6 diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java index e124e1101d147..5738d4650726c 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java @@ -106,6 +106,7 @@ public final class AdlConfKeys { "adl.feature.ownerandgroup.enableupn"; static final boolean ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT = false; public static final String ADL_HTTP_TIMEOUT = "adl.http.timeout"; + public static final String ADL_SSL_CHANNEL_MODE = "adl.ssl.channel.mode"; public static void addDeprecatedKeys() { Configuration.addDeprecations(new DeprecationDelta[]{ diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java index 79e8a698da141..3955721a765f1 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java @@ -203,6 +203,10 @@ public void initialize(URI storeUri, Configuration originalConf) LOG.info("No valid ADL SDK timeout configured: using SDK default."); } + String sslChannelMode = conf.get(ADL_SSL_CHANNEL_MODE, + "Default"); + options.setSSLChannelMode(sslChannelMode); + adlClient.setOptions(options); boolean trackLatency = conf diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/troubleshooting_adl.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/troubleshooting_adl.md index b362a9cb48802..0cc1a18c0bbba 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/troubleshooting_adl.md +++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/troubleshooting_adl.md @@ -153,3 +153,13 @@ addressed by lowering the timeout used by the SDK. A lower timeout at the storage layer may allow more retries to be attempted and actually increase the likelihood of success before hitting the framework's timeout, as attempts that may ultimately fail will fail faster. + +## SSL Socket Channel Mode + +ADL SDK will by default attempt to create secure socket connections over +OpenSSL as they provide significant performance improvements over Https. If +there are runtime issues, SDK will default connections over Default_JSE. This +can be overridden with the hadoop property `adl.ssl.channel.mode`. Possible +values for this config are OpenSSL, Default_JSE and Default (default). +Setting the config to OpenSSL or Default_JSE will try the connection to +only that mode. diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java index ca762d9315a46..61c063db695e7 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.adl.live; +import com.microsoft.azure.datalake.store.SSLSocketFactoryEx.SSLChannelMode; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.adl.AdlFileSystem; import org.junit.Assert; @@ -29,6 +30,7 @@ import java.net.URISyntaxException; import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_HTTP_TIMEOUT; +import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_SSL_CHANNEL_MODE; /** * Tests interactions with SDK and ensures configuration is having the desired @@ -53,7 +55,6 @@ public void testDefaultTimeout() throws IOException { // Skip this test if we can't get a real FS Assume.assumeNotNull(fs); - effectiveTimeout = fs.getAdlClient().getDefaultTimeout(); Assert.assertFalse("A negative timeout is not supposed to take effect", effectiveTimeout < 0); @@ -74,4 +75,67 @@ public void testDefaultTimeout() throws IOException { // The default value may vary by SDK, so that value is not tested here. } + + @Test + public void testSSLChannelMode() throws IOException { + AdlFileSystem fs = null; + Configuration conf = null; + + conf = AdlStorageConfiguration.getConfiguration(); + conf.set(ADL_SSL_CHANNEL_MODE, "OpenSSl"); + try { + fs = (AdlFileSystem) + (AdlStorageConfiguration.createStorageConnector(conf)); + } catch (URISyntaxException e) { + throw new IllegalStateException("ADL FileSystem initialization failed. " + + "Please check test.fs.adl.name property.", e); + } + + SSLChannelMode sslChannelMode = fs.getAdlClient().getSSLChannelMode(); + Assert.assertTrue("Channel mode needs to be OpenSSL", + sslChannelMode == SSLChannelMode.OpenSSL); + + conf = AdlStorageConfiguration.getConfiguration(); + conf.set(ADL_SSL_CHANNEL_MODE, "Default_JSE"); + try { + fs = (AdlFileSystem) + (AdlStorageConfiguration.createStorageConnector(conf)); + } catch (URISyntaxException e) { + throw new IllegalStateException("Can not initialize ADL FileSystem. " + + "Please check test.fs.adl.name property.", e); + } + + sslChannelMode = fs.getAdlClient().getSSLChannelMode(); + Assert.assertTrue("Channel mode needs to be Default_JSE", + sslChannelMode == SSLChannelMode.Default_JSE); + + conf = AdlStorageConfiguration.getConfiguration(); + conf.set(ADL_SSL_CHANNEL_MODE, "Default"); + try { + fs = (AdlFileSystem) + (AdlStorageConfiguration.createStorageConnector(conf)); + } catch (URISyntaxException e) { + throw new IllegalStateException("Can not initialize ADL FileSystem. " + + "Please check test.fs.adl.name property.", e); + } + + sslChannelMode = fs.getAdlClient().getSSLChannelMode(); + Assert.assertTrue("Channel mode needs to be Default", + sslChannelMode == SSLChannelMode.Default); + + conf = AdlStorageConfiguration.getConfiguration(); + conf.set(ADL_SSL_CHANNEL_MODE, "Invalid"); + try { + fs = (AdlFileSystem) + (AdlStorageConfiguration.createStorageConnector(conf)); + } catch (URISyntaxException e) { + throw new IllegalStateException("Can not initialize ADL FileSystem. " + + "Please check test.fs.adl.name property.", e); + } + + sslChannelMode = fs.getAdlClient().getSSLChannelMode(); + Assert.assertTrue("Channel mode needs to be Default when adl.ssl" + + ".channel.mode config is missing or is invalid", + sslChannelMode == SSLChannelMode.Default); + } } From 92d4cdc1ef0c1dfdef07314f814c5babedf9b3e4 Mon Sep 17 00:00:00 2001 From: Sneha Vijayarajan Date: Wed, 4 Sep 2019 13:33:40 +0530 Subject: [PATCH 2/3] Incorporating review comments --- .../src/main/resources/core-default.xml | 9 ++- .../fs/adl/live/TestAdlSdkConfiguration.java | 74 ++++++------------- 2 files changed, 27 insertions(+), 56 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 920d4c9ce7971..583f833dbeb84 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -3377,9 +3377,12 @@ adl.ssl.channel.mode - When OpenSSL - SSL socket connections are created in OpenSSL mode. - When Default_JSE - SSL socket connections are created in the default JSE mode. - When Default (default) - SSL socket connections are attempted with OpenSSL + Valid inputs are OpenSSL, Default_JSE and Default (case insensitive). + If config is missing or is invalid, SSL Channel mode will be set to Default. + + When OpenSSL, SSL socket connections are created in OpenSSL mode. + When Default_JSE, SSL socket connections are created in the default JSE mode. + When Default, SSL socket connections are attempted with OpenSSL and will fallback to Default_JSE mode if OpenSSL is not available at runtime. diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java index 61c063db695e7..954bbc0de6841 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java @@ -20,6 +20,7 @@ package org.apache.hadoop.fs.adl.live; import com.microsoft.azure.datalake.store.SSLSocketFactoryEx.SSLChannelMode; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.adl.AdlFileSystem; import org.junit.Assert; @@ -77,65 +78,32 @@ public void testDefaultTimeout() throws IOException { } @Test - public void testSSLChannelMode() throws IOException { + public void testSSLChannelModeConfig() + throws IOException, URISyntaxException { + testSSLChannelMode(SSLChannelMode.OpenSSL, "OpenSSL"); + testSSLChannelMode(SSLChannelMode.Default_JSE, "Default_JSE"); + testSSLChannelMode(SSLChannelMode.Default, "Default"); + // If config set is invalid, SSL channel mode will be Default. + testSSLChannelMode(SSLChannelMode.Default, "Invalid"); + // Config value is case insensitive. + testSSLChannelMode(SSLChannelMode.OpenSSL, "openssl"); + } + + public void testSSLChannelMode(SSLChannelMode expectedMode, + String sslChannelModeConfigValue) throws IOException, URISyntaxException { + AdlFileSystem fs = null; Configuration conf = null; conf = AdlStorageConfiguration.getConfiguration(); - conf.set(ADL_SSL_CHANNEL_MODE, "OpenSSl"); - try { - fs = (AdlFileSystem) - (AdlStorageConfiguration.createStorageConnector(conf)); - } catch (URISyntaxException e) { - throw new IllegalStateException("ADL FileSystem initialization failed. " - + "Please check test.fs.adl.name property.", e); - } + conf.set(ADL_SSL_CHANNEL_MODE, sslChannelModeConfigValue); + fs = (AdlFileSystem) (AdlStorageConfiguration.createStorageConnector(conf)); SSLChannelMode sslChannelMode = fs.getAdlClient().getSSLChannelMode(); - Assert.assertTrue("Channel mode needs to be OpenSSL", - sslChannelMode == SSLChannelMode.OpenSSL); - - conf = AdlStorageConfiguration.getConfiguration(); - conf.set(ADL_SSL_CHANNEL_MODE, "Default_JSE"); - try { - fs = (AdlFileSystem) - (AdlStorageConfiguration.createStorageConnector(conf)); - } catch (URISyntaxException e) { - throw new IllegalStateException("Can not initialize ADL FileSystem. " - + "Please check test.fs.adl.name property.", e); - } - - sslChannelMode = fs.getAdlClient().getSSLChannelMode(); - Assert.assertTrue("Channel mode needs to be Default_JSE", - sslChannelMode == SSLChannelMode.Default_JSE); - - conf = AdlStorageConfiguration.getConfiguration(); - conf.set(ADL_SSL_CHANNEL_MODE, "Default"); - try { - fs = (AdlFileSystem) - (AdlStorageConfiguration.createStorageConnector(conf)); - } catch (URISyntaxException e) { - throw new IllegalStateException("Can not initialize ADL FileSystem. " - + "Please check test.fs.adl.name property.", e); - } - - sslChannelMode = fs.getAdlClient().getSSLChannelMode(); - Assert.assertTrue("Channel mode needs to be Default", - sslChannelMode == SSLChannelMode.Default); - - conf = AdlStorageConfiguration.getConfiguration(); - conf.set(ADL_SSL_CHANNEL_MODE, "Invalid"); - try { - fs = (AdlFileSystem) - (AdlStorageConfiguration.createStorageConnector(conf)); - } catch (URISyntaxException e) { - throw new IllegalStateException("Can not initialize ADL FileSystem. " - + "Please check test.fs.adl.name property.", e); - } + Assert.assertTrue( + "Effective SSL Channel Mode : " + sslChannelMode.toString() + " is" + + " unexpected when config adl.ssl.channel.mode is set to : " + + sslChannelModeConfigValue, sslChannelMode == expectedMode); - sslChannelMode = fs.getAdlClient().getSSLChannelMode(); - Assert.assertTrue("Channel mode needs to be Default when adl.ssl" - + ".channel.mode config is missing or is invalid", - sslChannelMode == SSLChannelMode.Default); } } From 73c75a2343b68812f29d19bbd4a85700429585bd Mon Sep 17 00:00:00 2001 From: Sneha Vijayarajan Date: Wed, 4 Sep 2019 13:54:31 +0530 Subject: [PATCH 3/3] Modify test assertTrue to assertEquals --- .../hadoop/fs/adl/live/TestAdlSdkConfiguration.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java index 954bbc0de6841..980b683f66e41 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java @@ -100,10 +100,8 @@ public void testSSLChannelMode(SSLChannelMode expectedMode, fs = (AdlFileSystem) (AdlStorageConfiguration.createStorageConnector(conf)); SSLChannelMode sslChannelMode = fs.getAdlClient().getSSLChannelMode(); - Assert.assertTrue( - "Effective SSL Channel Mode : " + sslChannelMode.toString() + " is" - + " unexpected when config adl.ssl.channel.mode is set to : " - + sslChannelModeConfigValue, sslChannelMode == expectedMode); - + Assert.assertEquals( + "Unexpected SSL Channel Mode for adl.ssl.channel.mode config value : " + + sslChannelModeConfigValue, expectedMode, sslChannelMode); } }