From e0b34b31c682c24cd049910e0c4e8ba63213e4ea Mon Sep 17 00:00:00 2001 From: sumangala Date: Wed, 31 Mar 2021 18:20:45 +0530 Subject: [PATCH 01/15] mask oids --- .../azurebfs/services/AbfsHttpOperation.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java index 720b99b9f8390..eb31281ef6c44 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java @@ -52,6 +52,7 @@ public class AbfsHttpOperation implements AbfsPerfLoggable { private static final Logger LOG = LoggerFactory.getLogger(AbfsHttpOperation.class); public static final String SIGNATURE_QUERY_PARAM_KEY = "sig="; + public static final String[] SAS_OID_PARAM_KEYS = {"skoid", "saoid", "suoid"}; private static final int CONNECT_TIMEOUT = 30 * 1000; private static final int READ_TIMEOUT = 30 * 1000; @@ -547,6 +548,7 @@ public static String encodedUrlStr(String url) { public String getSignatureMaskedUrl() { if (this.maskedUrl == null) { this.maskedUrl = getSignatureMaskedUrl(this.url.toString()); + maskSASObjectIDs(); } return this.maskedUrl; } @@ -558,6 +560,24 @@ public String getSignatureMaskedEncodedUrl() { return this.maskedEncodedUrl; } + public void maskSASObjectIDs() { + int oidStartIdx, ampIdx, oidEndIndex, qpStrIdx; + for (String qpKey : SAS_OID_PARAM_KEYS) { + qpStrIdx = maskedUrl.indexOf('&' + qpKey); + if (qpStrIdx == -1) { + qpStrIdx = maskedUrl.indexOf('?' + qpKey); + if (qpStrIdx == -1) { + continue; + } + } + oidStartIdx = qpStrIdx + qpKey.length() + 1; + ampIdx = maskedUrl.indexOf("&", oidStartIdx); + oidEndIndex = (ampIdx != -1) ? ampIdx : maskedUrl.length(); + maskedUrl = maskedUrl.substring(0, oidStartIdx + 5) + "XXXX" + maskedUrl + .substring(oidEndIndex); + } + } + public static class AbfsHttpOperationWithFixedResult extends AbfsHttpOperation { /** * Creates an instance to represent fixed results. From dc641124c903e8650f8512f91ed0b31636b78012 Mon Sep 17 00:00:00 2001 From: sumangala Date: Wed, 31 Mar 2021 19:03:43 +0530 Subject: [PATCH 02/15] basic test --- ...ITestAzureBlobFileSystemDelegationSAS.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java index 50ce257b4a844..f543418cd50fd 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java @@ -64,6 +64,7 @@ */ public class ITestAzureBlobFileSystemDelegationSAS extends AbstractAbfsIntegrationTest { private static final String TEST_GROUP = UUID.randomUUID().toString(); + private static final String[] OID_KEYS = {"skoid", "saoid", "suoid"}; private static final Logger LOG = LoggerFactory.getLogger(ITestAzureBlobFileSystemDelegationSAS.class); @@ -410,6 +411,31 @@ public void testSignatureMask() throws Exception { .startsWith("sig%3DXXXX"); } + @Test + public void testSASObjectIDMask() throws IOException { + final AzureBlobFileSystem fs = getFileSystem(); + AbfsHttpOperation abfsHttpOperation = fs.getAbfsClient().getAclStatus("/") + .getResult(); + String url = abfsHttpOperation.getSignatureMaskedUrl(); + String encodedUrl = abfsHttpOperation.getSignatureMaskedEncodedUrl(); + int startIndex; + for (String qpKey : OID_KEYS) { + startIndex = url.indexOf(qpKey); + if (startIndex != -1) { + Assertions.assertThat(url.substring(startIndex + qpKey.length() + 5)) + .describedAs("OID query param should be masked in url") + .startsWith("XXXX"); + } + startIndex = encodedUrl.indexOf(qpKey); + if (startIndex != -1) { + Assertions + .assertThat(encodedUrl.substring(startIndex + qpKey.length() + 7)) + .describedAs("OID query param should be masked in encoded url") + .startsWith("XXXX"); + } + } + } + @Test public void testSignatureMaskOnExceptionMessage() throws Exception { intercept(IOException.class, "sig=XXXX", From 8e3ea6ef829b4d3391b986d1de118907369adcff Mon Sep 17 00:00:00 2001 From: sumangala Date: Wed, 31 Mar 2021 20:52:59 +0530 Subject: [PATCH 03/15] spotbug viz --- .../apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java index eb31281ef6c44..cac4f3df39a09 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java @@ -52,7 +52,7 @@ public class AbfsHttpOperation implements AbfsPerfLoggable { private static final Logger LOG = LoggerFactory.getLogger(AbfsHttpOperation.class); public static final String SIGNATURE_QUERY_PARAM_KEY = "sig="; - public static final String[] SAS_OID_PARAM_KEYS = {"skoid", "saoid", "suoid"}; + private static final String[] SAS_OID_PARAM_KEYS = {"skoid", "saoid", "suoid"}; private static final int CONNECT_TIMEOUT = 30 * 1000; private static final int READ_TIMEOUT = 30 * 1000; From 57c437e76e4efbc4198b158f8508cf8bca9f9b77 Mon Sep 17 00:00:00 2001 From: sumangala Date: Fri, 9 Apr 2021 11:24:12 +0530 Subject: [PATCH 04/15] move to utils --- .../AbfsRestOperationException.java | 4 +- .../azurebfs/services/AbfsHttpOperation.java | 82 +++++-------------- .../hadoop/fs/azurebfs/utils/UriUtils.java | 43 ++++++++++ ...ITestAzureBlobFileSystemDelegationSAS.java | 14 ++-- .../services/TestAbfsHttpOperation.java | 6 +- 5 files changed, 75 insertions(+), 74 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/AbfsRestOperationException.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/AbfsRestOperationException.java index 61b625657c14e..6c53762363840 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/AbfsRestOperationException.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/AbfsRestOperationException.java @@ -87,7 +87,7 @@ private static String formatMessage(final AbfsHttpOperation abfsHttpOperation) { "Operation failed: \"%1$s\", %2$s, HEAD, %3$s", abfsHttpOperation.getStatusDescription(), abfsHttpOperation.getStatusCode(), - abfsHttpOperation.getSignatureMaskedUrl()); + abfsHttpOperation.getMaskedUrl()); } return String.format( @@ -95,7 +95,7 @@ private static String formatMessage(final AbfsHttpOperation abfsHttpOperation) { abfsHttpOperation.getStatusDescription(), abfsHttpOperation.getStatusCode(), abfsHttpOperation.getMethod(), - abfsHttpOperation.getSignatureMaskedUrl(), + abfsHttpOperation.getMaskedUrl(), abfsHttpOperation.getStorageErrorCode(), // Remove break line to ensure the request id and timestamp can be shown in console. abfsHttpOperation.getStorageErrorMessage().replaceAll("\\n", " ")); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java index cac4f3df39a09..3a67235dd8dfe 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java @@ -21,16 +21,15 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.UnsupportedEncodingException; import java.net.HttpURLConnection; import java.net.URL; -import java.net.URLEncoder; import java.util.List; import java.util.UUID; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLSocketFactory; +import org.apache.hadoop.fs.azurebfs.utils.UriUtils; import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory; import org.codehaus.jackson.JsonFactory; import org.codehaus.jackson.JsonParser; @@ -51,9 +50,6 @@ public class AbfsHttpOperation implements AbfsPerfLoggable { private static final Logger LOG = LoggerFactory.getLogger(AbfsHttpOperation.class); - public static final String SIGNATURE_QUERY_PARAM_KEY = "sig="; - private static final String[] SAS_OID_PARAM_KEYS = {"skoid", "saoid", "suoid"}; - private static final int CONNECT_TIMEOUT = 30 * 1000; private static final int READ_TIMEOUT = 30 * 1000; @@ -195,7 +191,7 @@ public String toString() { sb.append(","); sb.append(method); sb.append(","); - sb.append(getSignatureMaskedUrl()); + sb.append(getMaskedUrl()); return sb.toString(); } @@ -228,11 +224,27 @@ public String getLogString() { .append(" m=") .append(method) .append(" u=") - .append(getSignatureMaskedEncodedUrl()); + .append(getMaskedEncodedUrl()); return sb.toString(); } + public String getMaskedUrl() { + if (maskedUrl != null) { + return maskedUrl; + } + return UriUtils.getMaskedUrl(url.toString()); + } + + public String getMaskedEncodedUrl() { + if (maskedEncodedUrl != null) { + return maskedEncodedUrl; + } + return maskedUrl == null ? + UriUtils.getMaskedEncodedUrl(url.toString()) : + UriUtils.encodedUrlStr(maskedUrl); + } + /** * Initializes a new HTTP request and opens the connection. * @@ -522,62 +534,6 @@ private boolean isNullInputStream(InputStream stream) { return stream == null ? true : false; } - public static String getSignatureMaskedUrl(String url) { - int qpStrIdx = url.indexOf('?' + SIGNATURE_QUERY_PARAM_KEY); - if (qpStrIdx == -1) { - qpStrIdx = url.indexOf('&' + SIGNATURE_QUERY_PARAM_KEY); - } - if (qpStrIdx == -1) { - return url; - } - final int sigStartIdx = qpStrIdx + SIGNATURE_QUERY_PARAM_KEY.length() + 1; - final int ampIdx = url.indexOf("&", sigStartIdx); - final int sigEndIndex = (ampIdx != -1) ? ampIdx : url.length(); - String signature = url.substring(sigStartIdx, sigEndIndex); - return url.replace(signature, "XXXX"); - } - - public static String encodedUrlStr(String url) { - try { - return URLEncoder.encode(url, "UTF-8"); - } catch (UnsupportedEncodingException e) { - return "https%3A%2F%2Ffailed%2Fto%2Fencode%2Furl"; - } - } - - public String getSignatureMaskedUrl() { - if (this.maskedUrl == null) { - this.maskedUrl = getSignatureMaskedUrl(this.url.toString()); - maskSASObjectIDs(); - } - return this.maskedUrl; - } - - public String getSignatureMaskedEncodedUrl() { - if (this.maskedEncodedUrl == null) { - this.maskedEncodedUrl = encodedUrlStr(getSignatureMaskedUrl()); - } - return this.maskedEncodedUrl; - } - - public void maskSASObjectIDs() { - int oidStartIdx, ampIdx, oidEndIndex, qpStrIdx; - for (String qpKey : SAS_OID_PARAM_KEYS) { - qpStrIdx = maskedUrl.indexOf('&' + qpKey); - if (qpStrIdx == -1) { - qpStrIdx = maskedUrl.indexOf('?' + qpKey); - if (qpStrIdx == -1) { - continue; - } - } - oidStartIdx = qpStrIdx + qpKey.length() + 1; - ampIdx = maskedUrl.indexOf("&", oidStartIdx); - oidEndIndex = (ampIdx != -1) ? ampIdx : maskedUrl.length(); - maskedUrl = maskedUrl.substring(0, oidStartIdx + 5) + "XXXX" + maskedUrl - .substring(oidEndIndex); - } - } - public static class AbfsHttpOperationWithFixedResult extends AbfsHttpOperation { /** * Creates an instance to represent fixed results. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java index 1bbc1b39e1689..1da1bdefbfd95 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java @@ -18,6 +18,8 @@ package org.apache.hadoop.fs.azurebfs.utils; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; import java.util.regex.Pattern; /** @@ -26,6 +28,9 @@ public final class UriUtils { private static final String ABFS_URI_REGEX = "[^.]+\\.dfs\\.(preprod\\.){0,1}core\\.windows\\.net"; private static final Pattern ABFS_URI_PATTERN = Pattern.compile(ABFS_URI_REGEX); + public static final String SIGNATURE_QUERY_PARAM_KEY = "sig="; + private static final String[] MASK_PARAM_KEYS = {"skoid=", "saoid=", "suoid=", + SIGNATURE_QUERY_PARAM_KEY}; /** * Checks whether a string includes abfs url. @@ -73,6 +78,44 @@ public static String generateUniqueTestPath() { return testUniqueForkId == null ? "/test" : "/" + testUniqueForkId + "/test"; } + public static String encodedUrlStr(String url) { + try { + return URLEncoder.encode(url, "UTF-8"); + } catch (UnsupportedEncodingException e) { + return "https%3A%2F%2Ffailed%2Fto%2Fencode%2Furl"; + } + } + + public static String getMaskedUrl(String url) { + return maskSigAndOIDs(url); + } + + public static String getMaskedEncodedUrl(String url) { + return encodedUrlStr(getMaskedUrl(url)); + } + + public static String maskSigAndOIDs(String url) { + for (String qpKey : MASK_PARAM_KEYS) { + int qpStrIdx = url.indexOf('&' + qpKey); + if (qpStrIdx == -1) { + qpStrIdx = url.indexOf('?' + qpKey); + if (qpStrIdx == -1) { + continue; + } + } + int startIdx = qpStrIdx + qpKey.length() + 1; + int ampIdx = url.indexOf("&", startIdx); + int endIndex = (ampIdx != -1) ? ampIdx : url.length(); + if (qpKey.equals(SIGNATURE_QUERY_PARAM_KEY)) { + String signature = url.substring(startIdx, endIndex); + url = url.replace(signature, "XXXX"); + } else { + url = url.substring(0, startIdx + 4) + "XXXX" + url.substring(endIndex); + } + } + return url; + } + private UriUtils() { } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java index f543418cd50fd..66bbcf1d091ea 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java @@ -64,7 +64,7 @@ */ public class ITestAzureBlobFileSystemDelegationSAS extends AbstractAbfsIntegrationTest { private static final String TEST_GROUP = UUID.randomUUID().toString(); - private static final String[] OID_KEYS = {"skoid", "saoid", "suoid"}; + private static final String[] OID_KEYS = {"skoid=", "saoid=", "suoid="}; private static final Logger LOG = LoggerFactory.getLogger(ITestAzureBlobFileSystemDelegationSAS.class); @@ -401,8 +401,8 @@ public void testSignatureMask() throws Exception { AbfsRestOperation abfsHttpRestOperation = fs.getAbfsClient() .renamePath(src, "/testABC" + "/abc.txt", null); AbfsHttpOperation result = abfsHttpRestOperation.getResult(); - String url = result.getSignatureMaskedUrl(); - String encodedUrl = result.getSignatureMaskedEncodedUrl(); + String url = result.getMaskedUrl(); + String encodedUrl = result.getMaskedEncodedUrl(); Assertions.assertThat(url.substring(url.indexOf("sig="))) .describedAs("Signature query param should be masked") .startsWith("sig=XXXX"); @@ -416,20 +416,20 @@ public void testSASObjectIDMask() throws IOException { final AzureBlobFileSystem fs = getFileSystem(); AbfsHttpOperation abfsHttpOperation = fs.getAbfsClient().getAclStatus("/") .getResult(); - String url = abfsHttpOperation.getSignatureMaskedUrl(); - String encodedUrl = abfsHttpOperation.getSignatureMaskedEncodedUrl(); + String url = abfsHttpOperation.getMaskedUrl(); + String encodedUrl = abfsHttpOperation.getMaskedEncodedUrl(); int startIndex; for (String qpKey : OID_KEYS) { startIndex = url.indexOf(qpKey); if (startIndex != -1) { - Assertions.assertThat(url.substring(startIndex + qpKey.length() + 5)) + Assertions.assertThat(url.substring(startIndex + qpKey.length() + 4)) .describedAs("OID query param should be masked in url") .startsWith("XXXX"); } startIndex = encodedUrl.indexOf(qpKey); if (startIndex != -1) { Assertions - .assertThat(encodedUrl.substring(startIndex + qpKey.length() + 7)) + .assertThat(encodedUrl.substring(startIndex + qpKey.length() + 6)) .describedAs("OID query param should be masked in encoded url") .startsWith("XXXX"); } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java index bb7059a7a5032..e71c47ac18afb 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java @@ -25,6 +25,8 @@ import org.assertj.core.api.Assertions; import org.junit.Test; +import org.apache.hadoop.fs.azurebfs.utils.UriUtils; + public class TestAbfsHttpOperation { @Test @@ -79,13 +81,13 @@ private void testIfMaskAndEncodeSuccessful(final String scenario, final String url, final String expectedMaskedUrl) throws UnsupportedEncodingException { - Assertions.assertThat(AbfsHttpOperation.getSignatureMaskedUrl(url)) + Assertions.assertThat(UriUtils.getMaskedUrl(url)) .describedAs(url + " (" + scenario + ") after masking should be: " + expectedMaskedUrl).isEqualTo(expectedMaskedUrl); final String expectedMaskedEncodedUrl = URLEncoder .encode(expectedMaskedUrl, "UTF-8"); - Assertions.assertThat(AbfsHttpOperation.encodedUrlStr(expectedMaskedUrl)) + Assertions.assertThat(UriUtils.encodedUrlStr(expectedMaskedUrl)) .describedAs( url + " (" + scenario + ") after masking and encoding should " + "be: " + expectedMaskedEncodedUrl) From 407143ff232959ada459122237b563478aaa3e8e Mon Sep 17 00:00:00 2001 From: sumangala Date: Fri, 9 Apr 2021 19:25:02 +0530 Subject: [PATCH 05/15] tests --- .../services/TestAbfsHttpOperation.java | 49 ++++++++++++------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java index e71c47ac18afb..d7ef7ed009fdb 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java @@ -35,46 +35,57 @@ public void testMaskingAndEncoding() testIfMaskAndEncodeSuccessful("Where sig is the only query param", "http://www.testurl.net?sig=abcd", "http://www.testurl.net?sig=XXXX"); + testIfMaskAndEncodeSuccessful("Where oid is the only query param", + "http://www.testurl.net?saoid=abcd", "http://www.testurl.net?sig=XXXX"); + testIfMaskAndEncodeSuccessful("Where sig is the first query param", "http://www.testurl.net?sig=abcd&abc=xyz", "http://www.testurl.net?sig=XXXX&abc=xyz"); + testIfMaskAndEncodeSuccessful("Where oid is the first query param", + "http://www.testurl.net?skoid=abcd123&abc=xyz", + "http://www.testurl.net?skoid=abcdXXXX&abc=xyz"); + testIfMaskAndEncodeSuccessful( - "Where sig is neither first nor last query param", - "http://www.testurl.net?lmn=abc&sig=abcd&abc=xyz", - "http://www.testurl.net?lmn=abc&sig=XXXX&abc=xyz"); + "Where sig/oid are neither first nor last query param", + "http://www.testurl.net?lmn=abc&sig=abcd&suoid=mnop789&abc=xyz", + "http://www.testurl.net?lmn=abc&sig=XXXX&suoid=mnopXXXX&abc=xyz"); testIfMaskAndEncodeSuccessful("Where sig is the last query param", "http://www.testurl.net?abc=xyz&sig=abcd", "http://www.testurl.net?abc=xyz&sig=XXXX"); - testIfMaskAndEncodeSuccessful("Where sig query param is not present", + testIfMaskAndEncodeSuccessful("Where oid is the last query param", + "http://www.testurl.net?abc=xyz&saoid=abcd456", + "http://www.testurl.net?abc=xyz&saoid=abcdXXXX"); + + testIfMaskAndEncodeSuccessful("Where sig/oid query param are not present", "http://www.testurl.net?abc=xyz", "http://www.testurl.net?abc=xyz"); testIfMaskAndEncodeSuccessful( - "Where sig query param is not present but mysig", - "http://www.testurl.net?abc=xyz&mysig=qwerty", - "http://www.testurl.net?abc=xyz&mysig=qwerty"); + "Where sig/oid query param are not present but mysig and myoid", + "http://www.testurl.net?abc=xyz&mysig=qwerty&mysaoid=uvw", + "http://www.testurl.net?abc=xyz&mysig=qwerty&mysaoid=uvw"); testIfMaskAndEncodeSuccessful( - "Where sig query param is not present but sigmy", - "http://www.testurl.net?abc=xyz&sigmy=qwerty", - "http://www.testurl.net?abc=xyz&sigmy=qwerty"); + "Where sig/oid query param is not present but sigmy and oidmy", + "http://www.testurl.net?abc=xyz&sigmy=qwerty&skoidmy=uvw", + "http://www.testurl.net?abc=xyz&sigmy=qwerty&skoidmy=uvw"); testIfMaskAndEncodeSuccessful( - "Where sig query param is not present but a " + "value sig", - "http://www.testurl.net?abc=xyz&mnop=sig", - "http://www.testurl.net?abc=xyz&mnop=sig"); + "Where sig/oid query param is not present but values sig and oid", + "http://www.testurl.net?abc=xyz&mnop=sig&pqr=saoid", + "http://www.testurl.net?abc=xyz&mnop=sig&pqr=saoid"); testIfMaskAndEncodeSuccessful( - "Where sig query param is not present but a " + "value ends with sig", - "http://www.testurl.net?abc=xyz&mnop=abcsig", - "http://www.testurl.net?abc=xyz&mnop=abcsig"); + "Where sig/oid query param is not present but a value ends with sig/oid", + "http://www.testurl.net?abc=xyzsaoid&mnop=abcsig", + "http://www.testurl.net?abc=xyzsaoid&mnop=abcsig"); testIfMaskAndEncodeSuccessful( - "Where sig query param is not present but a " + "value starts with sig", - "http://www.testurl.net?abc=xyz&mnop=sigabc", - "http://www.testurl.net?abc=xyz&mnop=sigabc"); + "Where sig/oid query param is not present but a value starts with sig/oid", + "http://www.testurl.net?abc=saoidxyz&mnop=sigabc", + "http://www.testurl.net?abc=saoidxyz&mnop=sigabc"); } private void testIfMaskAndEncodeSuccessful(final String scenario, From 7af9f80120f64b089e12f1c9d9feb4292d61dd38 Mon Sep 17 00:00:00 2001 From: sumangala Date: Sat, 10 Apr 2021 00:43:00 +0530 Subject: [PATCH 06/15] minor change --- .../azurebfs/services/AbfsHttpOperation.java | 6 ++--- .../services/TestAbfsHttpOperation.java | 23 +++++++------------ 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java index 3a67235dd8dfe..bf7aff7a90807 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java @@ -240,9 +240,9 @@ public String getMaskedEncodedUrl() { if (maskedEncodedUrl != null) { return maskedEncodedUrl; } - return maskedUrl == null ? - UriUtils.getMaskedEncodedUrl(url.toString()) : - UriUtils.encodedUrlStr(maskedUrl); + return maskedUrl == null + ? UriUtils.getMaskedEncodedUrl(url.toString()) + : UriUtils.encodedUrlStr(maskedUrl); } /** diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java index d7ef7ed009fdb..4beab9332c7ed 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java @@ -36,28 +36,21 @@ public void testMaskingAndEncoding() "http://www.testurl.net?sig=abcd", "http://www.testurl.net?sig=XXXX"); testIfMaskAndEncodeSuccessful("Where oid is the only query param", - "http://www.testurl.net?saoid=abcd", "http://www.testurl.net?sig=XXXX"); + "http://www.testurl.net?saoid=abcdef", + "http://www.testurl.net?saoid=abcdXXXX"); - testIfMaskAndEncodeSuccessful("Where sig is the first query param", - "http://www.testurl.net?sig=abcd&abc=xyz", - "http://www.testurl.net?sig=XXXX&abc=xyz"); - - testIfMaskAndEncodeSuccessful("Where oid is the first query param", - "http://www.testurl.net?skoid=abcd123&abc=xyz", - "http://www.testurl.net?skoid=abcdXXXX&abc=xyz"); + testIfMaskAndEncodeSuccessful("Where sig is the first query param, oid is last", + "http://www.testurl.net?sig=abcd&abc=xyz&saoid=abcd456", + "http://www.testurl.net?sig=XXXX&abc=xyz&saoid=abcdXXXX"); testIfMaskAndEncodeSuccessful( "Where sig/oid are neither first nor last query param", "http://www.testurl.net?lmn=abc&sig=abcd&suoid=mnop789&abc=xyz", "http://www.testurl.net?lmn=abc&sig=XXXX&suoid=mnopXXXX&abc=xyz"); - testIfMaskAndEncodeSuccessful("Where sig is the last query param", - "http://www.testurl.net?abc=xyz&sig=abcd", - "http://www.testurl.net?abc=xyz&sig=XXXX"); - - testIfMaskAndEncodeSuccessful("Where oid is the last query param", - "http://www.testurl.net?abc=xyz&saoid=abcd456", - "http://www.testurl.net?abc=xyz&saoid=abcdXXXX"); + testIfMaskAndEncodeSuccessful("Where sig is the last query param, oid is first", + "http://www.testurl.net?skoid=abcd123&abc=xyz&sig=abcd", + "http://www.testurl.net?skoid=abcdXXXX&abc=xyz&sig=XXXX"); testIfMaskAndEncodeSuccessful("Where sig/oid query param are not present", "http://www.testurl.net?abc=xyz", "http://www.testurl.net?abc=xyz"); From d72d490b27e5db691d2bcd3af4869b6d18f23295 Mon Sep 17 00:00:00 2001 From: sumangala Date: Sat, 10 Apr 2021 02:02:22 +0530 Subject: [PATCH 07/15] fix test assuming unique sig --- .../hadoop/fs/azurebfs/services/AbfsHttpOperation.java | 7 +++---- .../org/apache/hadoop/fs/azurebfs/utils/UriUtils.java | 8 -------- .../fs/azurebfs/services/TestAbfsHttpOperation.java | 8 ++++---- 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java index bf7aff7a90807..1b3774c6dbc88 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java @@ -233,16 +233,15 @@ public String getMaskedUrl() { if (maskedUrl != null) { return maskedUrl; } - return UriUtils.getMaskedUrl(url.toString()); + maskedUrl = UriUtils.getMaskedUrl(url.toString()); + return maskedUrl; } public String getMaskedEncodedUrl() { if (maskedEncodedUrl != null) { return maskedEncodedUrl; } - return maskedUrl == null - ? UriUtils.getMaskedEncodedUrl(url.toString()) - : UriUtils.encodedUrlStr(maskedUrl); + return UriUtils.encodedUrlStr(getMaskedUrl()); } /** diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java index 1da1bdefbfd95..7d00a8a683198 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java @@ -87,14 +87,6 @@ public static String encodedUrlStr(String url) { } public static String getMaskedUrl(String url) { - return maskSigAndOIDs(url); - } - - public static String getMaskedEncodedUrl(String url) { - return encodedUrlStr(getMaskedUrl(url)); - } - - public static String maskSigAndOIDs(String url) { for (String qpKey : MASK_PARAM_KEYS) { int qpStrIdx = url.indexOf('&' + qpKey); if (qpStrIdx == -1) { diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java index 4beab9332c7ed..215cb75da00d7 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java @@ -40,8 +40,8 @@ public void testMaskingAndEncoding() "http://www.testurl.net?saoid=abcdXXXX"); testIfMaskAndEncodeSuccessful("Where sig is the first query param, oid is last", - "http://www.testurl.net?sig=abcd&abc=xyz&saoid=abcd456", - "http://www.testurl.net?sig=XXXX&abc=xyz&saoid=abcdXXXX"); + "http://www.testurl.net?sig=abcd&abc=xyz&saoid=pqrs456", + "http://www.testurl.net?sig=XXXX&abc=xyz&saoid=pqrsXXXX"); testIfMaskAndEncodeSuccessful( "Where sig/oid are neither first nor last query param", @@ -49,8 +49,8 @@ public void testMaskingAndEncoding() "http://www.testurl.net?lmn=abc&sig=XXXX&suoid=mnopXXXX&abc=xyz"); testIfMaskAndEncodeSuccessful("Where sig is the last query param, oid is first", - "http://www.testurl.net?skoid=abcd123&abc=xyz&sig=abcd", - "http://www.testurl.net?skoid=abcdXXXX&abc=xyz&sig=XXXX"); + "http://www.testurl.net?skoid=pqrs123&abc=xyz&sig=abcd", + "http://www.testurl.net?skoid=pqrsXXXX&abc=xyz&sig=XXXX"); testIfMaskAndEncodeSuccessful("Where sig/oid query param are not present", "http://www.testurl.net?abc=xyz", "http://www.testurl.net?abc=xyz"); From 42d76d46af4fd505846fe291880ac92122f3d71d Mon Sep 17 00:00:00 2001 From: sumangala Date: Fri, 16 Apr 2021 13:00:41 +0530 Subject: [PATCH 08/15] temp --- .../azurebfs/services/AbfsHttpOperation.java | 15 ++-- .../hadoop/fs/azurebfs/utils/UriUtils.java | 89 ++++++++++++++----- .../services/TestAbfsHttpOperation.java | 5 +- 3 files changed, 77 insertions(+), 32 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java index 1b3774c6dbc88..edeef4e049f57 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java @@ -230,18 +230,19 @@ public String getLogString() { } public String getMaskedUrl() { - if (maskedUrl != null) { - return maskedUrl; + if (this.maskedUrl != null) { + return this.maskedUrl; } - maskedUrl = UriUtils.getMaskedUrl(url.toString()); - return maskedUrl; + this.maskedUrl = UriUtils.getMaskedUrl(this.url); + return this.maskedUrl; } public String getMaskedEncodedUrl() { - if (maskedEncodedUrl != null) { - return maskedEncodedUrl; + if (this.maskedEncodedUrl != null) { + return this.maskedEncodedUrl; } - return UriUtils.encodedUrlStr(getMaskedUrl()); + this.maskedEncodedUrl = UriUtils.encodedUrlStr(getMaskedUrl()); + return this.maskedEncodedUrl; } /** diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java index 7d00a8a683198..6be98a5b2c11b 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java @@ -19,18 +19,25 @@ package org.apache.hadoop.fs.azurebfs.utils; import java.io.UnsupportedEncodingException; +import java.net.URL; import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.regex.Pattern; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.AND_MARK; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EQUAL; + /** * Utility class to help with Abfs url transformation to blob urls. */ public final class UriUtils { private static final String ABFS_URI_REGEX = "[^.]+\\.dfs\\.(preprod\\.){0,1}core\\.windows\\.net"; private static final Pattern ABFS_URI_PATTERN = Pattern.compile(ABFS_URI_REGEX); - public static final String SIGNATURE_QUERY_PARAM_KEY = "sig="; - private static final String[] MASK_PARAM_KEYS = {"skoid=", "saoid=", "suoid=", - SIGNATURE_QUERY_PARAM_KEY}; + private static final ArrayList MASK_PARAM_KEYS = new ArrayList<>( + Arrays.asList("skoid=", "saoid=", "suoid=", "sig=")); + private static final String MASK = "XXXX"; /** * Checks whether a string includes abfs url. @@ -78,6 +85,37 @@ public static String generateUniqueTestPath() { return testUniqueForkId == null ? "/test" : "/" + testUniqueForkId + "/test"; } + public static List getUrlQueryParamKeyValueList(String query) { + String[] queryParameters = query.split(AND_MARK); + List keyValueList = new ArrayList<>(); + for (String parameter : queryParameters) { + keyValueList.add(new QueryKeyValuePair(parameter.split(EQUAL))); + } + return keyValueList; + } + + public static String maskUrlQueryParameters( + List keyValueList, + ArrayList queryParamsToMask) { + StringBuilder maskedUrl = new StringBuilder(); + for (QueryKeyValuePair keyValuePair : keyValueList) { + String key = keyValuePair.getKey(); + maskedUrl.append(key); + maskedUrl.append(EQUAL); + if (queryParamsToMask.contains(key + EQUAL)) { + if (key.contains("oid")) { //partial mask + maskedUrl.append(keyValuePair.getValue(), 0, 4); + } + maskedUrl.append(MASK); + } else { + maskedUrl.append(keyValuePair.getValue()); + } + maskedUrl.append(AND_MARK); + } + maskedUrl.deleteCharAt(maskedUrl.length() - 1); + return maskedUrl.toString(); + } + public static String encodedUrlStr(String url) { try { return URLEncoder.encode(url, "UTF-8"); @@ -86,28 +124,33 @@ public static String encodedUrlStr(String url) { } } - public static String getMaskedUrl(String url) { - for (String qpKey : MASK_PARAM_KEYS) { - int qpStrIdx = url.indexOf('&' + qpKey); - if (qpStrIdx == -1) { - qpStrIdx = url.indexOf('?' + qpKey); - if (qpStrIdx == -1) { - continue; - } - } - int startIdx = qpStrIdx + qpKey.length() + 1; - int ampIdx = url.indexOf("&", startIdx); - int endIndex = (ampIdx != -1) ? ampIdx : url.length(); - if (qpKey.equals(SIGNATURE_QUERY_PARAM_KEY)) { - String signature = url.substring(startIdx, endIndex); - url = url.replace(signature, "XXXX"); - } else { - url = url.substring(0, startIdx + 4) + "XXXX" + url.substring(endIndex); - } - } - return url; + public static String getMaskedUrl(URL url) { + String queryString = url.getQuery(); + List queryKeyValueList = getUrlQueryParamKeyValueList( + queryString); + String maskedQueryString = maskUrlQueryParameters(queryKeyValueList, + MASK_PARAM_KEYS); + return url.toString().replace(queryString, maskedQueryString); } private UriUtils() { } } + +class QueryKeyValuePair { + private final String key; + private final String value; + + public QueryKeyValuePair(String[] kvPair) { + key = kvPair[0]; + value = kvPair[1]; + } + + String getKey() { + return key; + } + + String getValue() { + return value; + } +} diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java index 215cb75da00d7..e0a9ac7675d87 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java @@ -20,6 +20,7 @@ import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; +import java.net.URL; import java.net.URLEncoder; import org.assertj.core.api.Assertions; @@ -83,9 +84,9 @@ public void testMaskingAndEncoding() private void testIfMaskAndEncodeSuccessful(final String scenario, final String url, final String expectedMaskedUrl) - throws UnsupportedEncodingException { + throws UnsupportedEncodingException, MalformedURLException { - Assertions.assertThat(UriUtils.getMaskedUrl(url)) + Assertions.assertThat(UriUtils.getMaskedUrl(new URL(url))) .describedAs(url + " (" + scenario + ") after masking should be: " + expectedMaskedUrl).isEqualTo(expectedMaskedUrl); From 3a95de32d14c0c9b5b00a91edd200cb1aa9991ef Mon Sep 17 00:00:00 2001 From: sumangala Date: Tue, 20 Apr 2021 01:51:16 +0530 Subject: [PATCH 09/15] tests --- .../hadoop/fs/azurebfs/utils/UriUtils.java | 78 ++++++++----------- .../services/TestAbfsHttpOperation.java | 14 ++++ .../fs/azurebfs/utils/TestUriUtils.java | 75 ++++++++++++++++++ 3 files changed, 123 insertions(+), 44 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java index 6be98a5b2c11b..ed9c0e7fa4605 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java @@ -21,11 +21,16 @@ import java.io.UnsupportedEncodingException; import java.net.URL; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.regex.Pattern; +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URLEncodedUtils; + import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.AND_MARK; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EQUAL; @@ -35,8 +40,11 @@ public final class UriUtils { private static final String ABFS_URI_REGEX = "[^.]+\\.dfs\\.(preprod\\.){0,1}core\\.windows\\.net"; private static final Pattern ABFS_URI_PATTERN = Pattern.compile(ABFS_URI_REGEX); - private static final ArrayList MASK_PARAM_KEYS = new ArrayList<>( - Arrays.asList("skoid=", "saoid=", "suoid=", "sig=")); + private static final ArrayList FULL_MASK_PARAM_KEYS = new ArrayList<>( + Collections.singleton("sig=")); + private static final ArrayList PARTIAL_MASK_PARAM_KEYS = + new ArrayList<>( + Arrays.asList("skoid=", "saoid=", "suoid=")); private static final String MASK = "XXXX"; /** @@ -85,30 +93,30 @@ public static String generateUniqueTestPath() { return testUniqueForkId == null ? "/test" : "/" + testUniqueForkId + "/test"; } - public static List getUrlQueryParamKeyValueList(String query) { - String[] queryParameters = query.split(AND_MARK); - List keyValueList = new ArrayList<>(); - for (String parameter : queryParameters) { - keyValueList.add(new QueryKeyValuePair(parameter.split(EQUAL))); - } - return keyValueList; - } - - public static String maskUrlQueryParameters( - List keyValueList, - ArrayList queryParamsToMask) { + public static String maskUrlQueryParameters(List keyValueList, + ArrayList queryParamsForFullMask, + ArrayList queryParamsForPartialMask) { StringBuilder maskedUrl = new StringBuilder(); - for (QueryKeyValuePair keyValuePair : keyValueList) { - String key = keyValuePair.getKey(); + for (NameValuePair keyValuePair : keyValueList) { + String key = keyValuePair.getName(); + if (!key.isEmpty()) { + key += EQUAL; + } else { + throw new IllegalArgumentException("Query param key can not be empty"); + } + + String value = keyValuePair.getValue(); maskedUrl.append(key); - maskedUrl.append(EQUAL); - if (queryParamsToMask.contains(key + EQUAL)) { - if (key.contains("oid")) { //partial mask - maskedUrl.append(keyValuePair.getValue(), 0, 4); + if (value != null && !value.isEmpty()) { //no mask + if (queryParamsForFullMask.contains(key)) { + maskedUrl.append(MASK); + } else if (queryParamsForPartialMask.contains(key)) { + int visibleLen = Math.min(4, value.length()); + maskedUrl.append(value, 0, visibleLen); + maskedUrl.append(MASK); + } else { + maskedUrl.append(value); } - maskedUrl.append(MASK); - } else { - maskedUrl.append(keyValuePair.getValue()); } maskedUrl.append(AND_MARK); } @@ -126,31 +134,13 @@ public static String encodedUrlStr(String url) { public static String getMaskedUrl(URL url) { String queryString = url.getQuery(); - List queryKeyValueList = getUrlQueryParamKeyValueList( - queryString); + List queryKeyValueList = URLEncodedUtils + .parse(String.valueOf(queryString), StandardCharsets.UTF_8); String maskedQueryString = maskUrlQueryParameters(queryKeyValueList, - MASK_PARAM_KEYS); + FULL_MASK_PARAM_KEYS, PARTIAL_MASK_PARAM_KEYS); return url.toString().replace(queryString, maskedQueryString); } private UriUtils() { } } - -class QueryKeyValuePair { - private final String key; - private final String value; - - public QueryKeyValuePair(String[] kvPair) { - key = kvPair[0]; - value = kvPair[1]; - } - - String getKey() { - return key; - } - - String getValue() { - return value; - } -} diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java index e0a9ac7675d87..5b23bf26deecc 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java @@ -82,6 +82,20 @@ public void testMaskingAndEncoding() "http://www.testurl.net?abc=saoidxyz&mnop=sigabc"); } + @Test + public void testUrlWithNullValues() + throws MalformedURLException, UnsupportedEncodingException { + testIfMaskAndEncodeSuccessful("Where param to be masked has null value", + "http://www.testurl.net?abc=xyz&saoid=&mnop=abcsig", + "http://www.testurl.net?abc=xyz&saoid=&mnop=abcsig"); + testIfMaskAndEncodeSuccessful("Where visible param has null value", + "http://www.testurl.net?abc=xyz&pqr=&mnop=abcd", + "http://www.testurl.net?abc=xyz&pqr=&mnop=abcd"); + testIfMaskAndEncodeSuccessful("Where last param has null value", + "http://www.testurl.net?abc=xyz&pqr=&mnop=", + "http://www.testurl.net?abc=xyz&pqr=&mnop="); + } + private void testIfMaskAndEncodeSuccessful(final String scenario, final String url, final String expectedMaskedUrl) throws UnsupportedEncodingException, MalformedURLException { diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java index 690e56c510559..cee6dd4539be1 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java @@ -18,9 +18,20 @@ package org.apache.hadoop.fs.azurebfs.utils; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + import org.junit.Assert; import org.junit.Test; +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URLEncodedUtils; +import org.apache.http.message.BasicNameValuePair; + +import static org.apache.hadoop.test.LambdaTestUtils.intercept; + /** * Test ABFS UriUtils. */ @@ -45,4 +56,68 @@ public void testExtractRawAccountName() throws Exception { Assert.assertEquals(null, UriUtils.extractAccountNameFromHostName(null)); Assert.assertEquals(null, UriUtils.extractAccountNameFromHostName("abfs.dfs.cores.windows.net")); } + + @Test + public void testMaskUrlQueryParameters() throws Exception { + ArrayList fullMask = new ArrayList<>(Arrays.asList("abc=", "bcd=")); + ArrayList partialMask = new ArrayList<>( + Arrays.asList("pqr=", "xyz=")); + + //Partial and full masking test + List keyValueList = URLEncodedUtils + .parse("abc=123&pqr=456&def=789&bcd=012&xyz=678", StandardCharsets.UTF_8); + Assert.assertEquals("Incorrect masking", + UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask), + "abc=XXXX&pqr=456XXXX&def=789&bcd=XXXX&xyz=678XXXX"); + + //For params entered for both full and partial masks, full mask applies + partialMask.add("abc="); + Assert.assertEquals("Full mask should apply", + UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask), + "abc=XXXX&pqr=456XXXX&def=789&bcd=XXXX&xyz=678XXXX"); + + //Duplicate key (to be masked) with different values + keyValueList = URLEncodedUtils + .parse("abc=123&pqr=456&abc=789", StandardCharsets.UTF_8); + Assert.assertEquals("Duplicate key: Both values should get masked", + UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask), + "abc=XXXX&pqr=456XXXX&abc=XXXX"); + + //Duplicate key (not to be masked) with different values + keyValueList = URLEncodedUtils + .parse("abc=123&def=456&pqr=789&def=000&s=1", StandardCharsets.UTF_8); + Assert.assertEquals("Duplicate key: no value should get masked", + UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask), + "abc=XXXX&def=456&pqr=789XXXX&def=000&s=1"); + + //Empty param value + keyValueList = URLEncodedUtils + .parse("abc=123&def=&pqr=789&s=1", StandardCharsets.UTF_8); + Assert.assertEquals("Incorrect url with empty query value", + "abc=XXXX&def=&pqr=789XXXX&s=1", + UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask)); + + //Empty param key + keyValueList = URLEncodedUtils + .parse("def=2&pqr=789&s=1", StandardCharsets.UTF_8); + keyValueList.add(new BasicNameValuePair("", "m1")); + List finalKeyValueList = keyValueList; + intercept(IllegalArgumentException.class, () -> UriUtils + .maskUrlQueryParameters(finalKeyValueList, fullMask, partialMask)); + + //Param (not to be masked) with null value + keyValueList = URLEncodedUtils + .parse("abc=123&pqr=789&s=1", StandardCharsets.UTF_8); + keyValueList.add(new BasicNameValuePair("null1", null)); + Assert.assertEquals("Null value, incorrect query construction", + "abc=XXXX&pqr=789XXXX&s=1&null1=", + UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask)); + + //Param (to be masked) with null value + keyValueList.add(new BasicNameValuePair("null2", null)); + fullMask.add("null2"); + Assert.assertEquals("No mask should be added for null value", + UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask), + "abc=XXXX&pqr=789XXXX&s=1&null1=&null2="); //no mask + } } From d635f073fa800b0062ac027078de09b3a3aa805b Mon Sep 17 00:00:00 2001 From: sumangala Date: Tue, 20 Apr 2021 08:15:19 +0530 Subject: [PATCH 10/15] fix null query fail --- .../java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java index ed9c0e7fa4605..9d6747b3836f1 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java @@ -134,8 +134,11 @@ public static String encodedUrlStr(String url) { public static String getMaskedUrl(URL url) { String queryString = url.getQuery(); + if (queryString == null) { + return url.toString(); + } List queryKeyValueList = URLEncodedUtils - .parse(String.valueOf(queryString), StandardCharsets.UTF_8); + .parse(queryString, StandardCharsets.UTF_8); String maskedQueryString = maskUrlQueryParameters(queryKeyValueList, FULL_MASK_PARAM_KEYS, PARTIAL_MASK_PARAM_KEYS); return url.toString().replace(queryString, maskedQueryString); From 1e391437c058f691918877ee06ed5a6cc3e4d499 Mon Sep 17 00:00:00 2001 From: sumangala Date: Tue, 20 Apr 2021 09:12:50 +0530 Subject: [PATCH 11/15] rm = --- .../hadoop/fs/azurebfs/utils/UriUtils.java | 10 +++---- .../fs/azurebfs/utils/TestUriUtils.java | 28 ++++++++++--------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java index 9d6747b3836f1..10e59c20a509f 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java @@ -41,10 +41,10 @@ public final class UriUtils { private static final String ABFS_URI_REGEX = "[^.]+\\.dfs\\.(preprod\\.){0,1}core\\.windows\\.net"; private static final Pattern ABFS_URI_PATTERN = Pattern.compile(ABFS_URI_REGEX); private static final ArrayList FULL_MASK_PARAM_KEYS = new ArrayList<>( - Collections.singleton("sig=")); + Collections.singleton("sig")); private static final ArrayList PARTIAL_MASK_PARAM_KEYS = new ArrayList<>( - Arrays.asList("skoid=", "saoid=", "suoid=")); + Arrays.asList("skoid", "saoid", "suoid")); private static final String MASK = "XXXX"; /** @@ -99,14 +99,12 @@ public static String maskUrlQueryParameters(List keyValueList, StringBuilder maskedUrl = new StringBuilder(); for (NameValuePair keyValuePair : keyValueList) { String key = keyValuePair.getName(); - if (!key.isEmpty()) { - key += EQUAL; - } else { + if (key.isEmpty()) { throw new IllegalArgumentException("Query param key can not be empty"); } - String value = keyValuePair.getValue(); maskedUrl.append(key); + maskedUrl.append(EQUAL); if (value != null && !value.isEmpty()) { //no mask if (queryParamsForFullMask.contains(key)) { maskedUrl.append(MASK); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java index cee6dd4539be1..391c6cddcea5e 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java @@ -59,36 +59,37 @@ public void testExtractRawAccountName() throws Exception { @Test public void testMaskUrlQueryParameters() throws Exception { - ArrayList fullMask = new ArrayList<>(Arrays.asList("abc=", "bcd=")); + ArrayList fullMask = new ArrayList<>(Arrays.asList("abc", "bcd")); ArrayList partialMask = new ArrayList<>( - Arrays.asList("pqr=", "xyz=")); + Arrays.asList("pqr", "xyz")); //Partial and full masking test List keyValueList = URLEncodedUtils - .parse("abc=123&pqr=456&def=789&bcd=012&xyz=678", StandardCharsets.UTF_8); + .parse("abc=123&pqr=456&def=789&bcd=012&xyz=678", + StandardCharsets.UTF_8); Assert.assertEquals("Incorrect masking", - UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask), - "abc=XXXX&pqr=456XXXX&def=789&bcd=XXXX&xyz=678XXXX"); + "abc=XXXX&pqr=456XXXX&def=789&bcd=XXXX&xyz=678XXXX", + UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask)); //For params entered for both full and partial masks, full mask applies partialMask.add("abc="); Assert.assertEquals("Full mask should apply", - UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask), - "abc=XXXX&pqr=456XXXX&def=789&bcd=XXXX&xyz=678XXXX"); + "abc=XXXX&pqr=456XXXX&def=789&bcd=XXXX&xyz=678XXXX", + UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask)); //Duplicate key (to be masked) with different values keyValueList = URLEncodedUtils .parse("abc=123&pqr=456&abc=789", StandardCharsets.UTF_8); Assert.assertEquals("Duplicate key: Both values should get masked", - UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask), - "abc=XXXX&pqr=456XXXX&abc=XXXX"); + "abc=XXXX&pqr=456XXXX&abc=XXXX", + UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask)); //Duplicate key (not to be masked) with different values keyValueList = URLEncodedUtils .parse("abc=123&def=456&pqr=789&def=000&s=1", StandardCharsets.UTF_8); Assert.assertEquals("Duplicate key: no value should get masked", - UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask), - "abc=XXXX&def=456&pqr=789XXXX&def=000&s=1"); + "abc=XXXX&def=456&pqr=789XXXX&def=000&s=1", + UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask)); //Empty param value keyValueList = URLEncodedUtils @@ -117,7 +118,8 @@ public void testMaskUrlQueryParameters() throws Exception { keyValueList.add(new BasicNameValuePair("null2", null)); fullMask.add("null2"); Assert.assertEquals("No mask should be added for null value", - UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask), - "abc=XXXX&pqr=789XXXX&s=1&null1=&null2="); //no mask + "abc=XXXX&pqr=789XXXX&s=1&null1=&null2=", UriUtils + .maskUrlQueryParameters(keyValueList, fullMask, + partialMask)); //no mask } } From 43597189aa88707ae0cac43951a35b4c5df4e201 Mon Sep 17 00:00:00 2001 From: sumangala Date: Thu, 22 Apr 2021 12:41:43 +0530 Subject: [PATCH 12/15] pr revw --- .../azurebfs/constants/HttpQueryParams.java | 6 +++ .../azurebfs/services/AbfsHttpOperation.java | 8 ++++ .../azurebfs/services/AbfsRestOperation.java | 1 + .../hadoop/fs/azurebfs/utils/UriUtils.java | 45 ++++++++++++------- ...ITestAzureBlobFileSystemDelegationSAS.java | 26 ----------- .../services/TestAbfsHttpOperation.java | 8 ++-- .../fs/azurebfs/utils/TestUriUtils.java | 25 ++++++----- 7 files changed, 61 insertions(+), 58 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpQueryParams.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpQueryParams.java index 8a4ca90f35870..e9bb95cad21cd 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpQueryParams.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpQueryParams.java @@ -41,5 +41,11 @@ public final class HttpQueryParams { public static final String QUERY_PARAM_UPN = "upn"; public static final String QUERY_PARAM_BLOBTYPE = "blobtype"; + //query params for SAS + public static final String QUERY_PARAM_SAOID = "saoid"; + public static final String QUERY_PARAM_SKOID = "skoid"; + public static final String QUERY_PARAM_SUOID = "suoid"; + public static final String QUERY_PARAM_SIGNATURE = "sig"; + private HttpQueryParams() {} } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java index edeef4e049f57..dd30752948d64 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java @@ -82,6 +82,7 @@ public class AbfsHttpOperation implements AbfsPerfLoggable { private long connectionTimeMs; private long sendRequestTimeMs; private long recvResponseTimeMs; + private boolean shouldMask = false; public static AbfsHttpOperation getAbfsHttpOperationWithFixedResult( final URL url, @@ -147,6 +148,10 @@ public String getRequestId() { return requestId; } + public void setMaskForSAS() { + shouldMask = true; + } + public int getBytesSent() { return bytesSent; } @@ -230,6 +235,9 @@ public String getLogString() { } public String getMaskedUrl() { + if (!shouldMask) { + return url.toString(); + } if (this.maskedUrl != null) { return this.maskedUrl; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index 584b71f1ee5bf..80b9ed2be0ac0 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -222,6 +222,7 @@ private boolean executeHttpOperation(final int retryCount) throws AzureBlobFileS break; case SAS: // do nothing; the SAS token should already be appended to the query string + httpOperation.setMaskForSAS(); //mask sig/oid from url for logs break; case SharedKey: // sign the HTTP request diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java index 10e59c20a509f..8884c11510629 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java @@ -22,17 +22,22 @@ import java.net.URL; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.regex.Pattern; +import org.apache.commons.lang3.StringUtils; import org.apache.http.NameValuePair; import org.apache.http.client.utils.URLEncodedUtils; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.AND_MARK; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EQUAL; +import static org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams.QUERY_PARAM_SAOID; +import static org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams.QUERY_PARAM_SIGNATURE; +import static org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams.QUERY_PARAM_SKOID; +import static org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams.QUERY_PARAM_SUOID; /** * Utility class to help with Abfs url transformation to blob urls. @@ -40,12 +45,12 @@ public final class UriUtils { private static final String ABFS_URI_REGEX = "[^.]+\\.dfs\\.(preprod\\.){0,1}core\\.windows\\.net"; private static final Pattern ABFS_URI_PATTERN = Pattern.compile(ABFS_URI_REGEX); - private static final ArrayList FULL_MASK_PARAM_KEYS = new ArrayList<>( - Collections.singleton("sig")); - private static final ArrayList PARTIAL_MASK_PARAM_KEYS = - new ArrayList<>( - Arrays.asList("skoid", "saoid", "suoid")); - private static final String MASK = "XXXX"; + private static final HashSet FULL_MASK_PARAM_KEYS = new HashSet<>( + Collections.singleton(QUERY_PARAM_SIGNATURE)); + private static final HashSet PARTIAL_MASK_PARAM_KEYS = new HashSet<>( + Arrays.asList(QUERY_PARAM_SKOID, QUERY_PARAM_SAOID, QUERY_PARAM_SUOID)); + private static final String FULL_MASK = "XXXX"; + private static final Character CHAR_MASK = 'X'; /** * Checks whether a string includes abfs url. @@ -94,24 +99,32 @@ public static String generateUniqueTestPath() { } public static String maskUrlQueryParameters(List keyValueList, - ArrayList queryParamsForFullMask, - ArrayList queryParamsForPartialMask) { - StringBuilder maskedUrl = new StringBuilder(); + HashSet queryParamsForFullMask, + HashSet queryParamsForPartialMask) { + return maskUrlQueryParameters(keyValueList, queryParamsForFullMask, + queryParamsForPartialMask, 256); + } + + public static String maskUrlQueryParameters(List keyValueList, + HashSet queryParamsForFullMask, + HashSet queryParamsForPartialMask, int queryLen) { + StringBuilder maskedUrl = new StringBuilder(queryLen); for (NameValuePair keyValuePair : keyValueList) { String key = keyValuePair.getName(); if (key.isEmpty()) { - throw new IllegalArgumentException("Query param key can not be empty"); + throw new IllegalArgumentException("Query param key should not be empty"); } String value = keyValuePair.getValue(); maskedUrl.append(key); maskedUrl.append(EQUAL); if (value != null && !value.isEmpty()) { //no mask if (queryParamsForFullMask.contains(key)) { - maskedUrl.append(MASK); + maskedUrl.append(FULL_MASK); } else if (queryParamsForPartialMask.contains(key)) { - int visibleLen = Math.min(4, value.length()); - maskedUrl.append(value, 0, visibleLen); - maskedUrl.append(MASK); + int valueLen = value.length(); + int maskedLen = valueLen - Math.min(4, valueLen); + maskedUrl.append(StringUtils.repeat(CHAR_MASK, maskedLen)); + maskedUrl.append(value, maskedLen, valueLen); } else { maskedUrl.append(value); } @@ -138,7 +151,7 @@ public static String getMaskedUrl(URL url) { List queryKeyValueList = URLEncodedUtils .parse(queryString, StandardCharsets.UTF_8); String maskedQueryString = maskUrlQueryParameters(queryKeyValueList, - FULL_MASK_PARAM_KEYS, PARTIAL_MASK_PARAM_KEYS); + FULL_MASK_PARAM_KEYS, PARTIAL_MASK_PARAM_KEYS, queryString.length()); return url.toString().replace(queryString, maskedQueryString); } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java index 66bbcf1d091ea..36b950c57bdb4 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java @@ -64,7 +64,6 @@ */ public class ITestAzureBlobFileSystemDelegationSAS extends AbstractAbfsIntegrationTest { private static final String TEST_GROUP = UUID.randomUUID().toString(); - private static final String[] OID_KEYS = {"skoid=", "saoid=", "suoid="}; private static final Logger LOG = LoggerFactory.getLogger(ITestAzureBlobFileSystemDelegationSAS.class); @@ -411,31 +410,6 @@ public void testSignatureMask() throws Exception { .startsWith("sig%3DXXXX"); } - @Test - public void testSASObjectIDMask() throws IOException { - final AzureBlobFileSystem fs = getFileSystem(); - AbfsHttpOperation abfsHttpOperation = fs.getAbfsClient().getAclStatus("/") - .getResult(); - String url = abfsHttpOperation.getMaskedUrl(); - String encodedUrl = abfsHttpOperation.getMaskedEncodedUrl(); - int startIndex; - for (String qpKey : OID_KEYS) { - startIndex = url.indexOf(qpKey); - if (startIndex != -1) { - Assertions.assertThat(url.substring(startIndex + qpKey.length() + 4)) - .describedAs("OID query param should be masked in url") - .startsWith("XXXX"); - } - startIndex = encodedUrl.indexOf(qpKey); - if (startIndex != -1) { - Assertions - .assertThat(encodedUrl.substring(startIndex + qpKey.length() + 6)) - .describedAs("OID query param should be masked in encoded url") - .startsWith("XXXX"); - } - } - } - @Test public void testSignatureMaskOnExceptionMessage() throws Exception { intercept(IOException.class, "sig=XXXX", diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java index 5b23bf26deecc..5e8fb9a08f5af 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java @@ -38,20 +38,20 @@ public void testMaskingAndEncoding() testIfMaskAndEncodeSuccessful("Where oid is the only query param", "http://www.testurl.net?saoid=abcdef", - "http://www.testurl.net?saoid=abcdXXXX"); + "http://www.testurl.net?saoid=XXcdef"); testIfMaskAndEncodeSuccessful("Where sig is the first query param, oid is last", "http://www.testurl.net?sig=abcd&abc=xyz&saoid=pqrs456", - "http://www.testurl.net?sig=XXXX&abc=xyz&saoid=pqrsXXXX"); + "http://www.testurl.net?sig=XXXX&abc=xyz&saoid=XXXs456"); testIfMaskAndEncodeSuccessful( "Where sig/oid are neither first nor last query param", "http://www.testurl.net?lmn=abc&sig=abcd&suoid=mnop789&abc=xyz", - "http://www.testurl.net?lmn=abc&sig=XXXX&suoid=mnopXXXX&abc=xyz"); + "http://www.testurl.net?lmn=abc&sig=XXXX&suoid=XXXp789&abc=xyz"); testIfMaskAndEncodeSuccessful("Where sig is the last query param, oid is first", "http://www.testurl.net?skoid=pqrs123&abc=xyz&sig=abcd", - "http://www.testurl.net?skoid=pqrsXXXX&abc=xyz&sig=XXXX"); + "http://www.testurl.net?skoid=XXXs123&abc=xyz&sig=XXXX"); testIfMaskAndEncodeSuccessful("Where sig/oid query param are not present", "http://www.testurl.net?abc=xyz", "http://www.testurl.net?abc=xyz"); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java index 391c6cddcea5e..8b8925ca76e08 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java @@ -21,6 +21,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import org.junit.Assert; @@ -59,43 +60,43 @@ public void testExtractRawAccountName() throws Exception { @Test public void testMaskUrlQueryParameters() throws Exception { - ArrayList fullMask = new ArrayList<>(Arrays.asList("abc", "bcd")); - ArrayList partialMask = new ArrayList<>( + HashSet fullMask = new HashSet<>(Arrays.asList("abc", "bcd")); + HashSet partialMask = new HashSet<>( Arrays.asList("pqr", "xyz")); //Partial and full masking test List keyValueList = URLEncodedUtils - .parse("abc=123&pqr=456&def=789&bcd=012&xyz=678", + .parse("abc=123&pqr=45678&def=789&bcd=012&xyz=678", StandardCharsets.UTF_8); Assert.assertEquals("Incorrect masking", - "abc=XXXX&pqr=456XXXX&def=789&bcd=XXXX&xyz=678XXXX", + "abc=XXXX&pqr=X5678&def=789&bcd=XXXX&xyz=678", UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask)); //For params entered for both full and partial masks, full mask applies - partialMask.add("abc="); + partialMask.add("abc"); Assert.assertEquals("Full mask should apply", - "abc=XXXX&pqr=456XXXX&def=789&bcd=XXXX&xyz=678XXXX", + "abc=XXXX&pqr=X5678&def=789&bcd=XXXX&xyz=678", UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask)); //Duplicate key (to be masked) with different values keyValueList = URLEncodedUtils - .parse("abc=123&pqr=456&abc=789", StandardCharsets.UTF_8); + .parse("abc=123&pqr=4561234&abc=789", StandardCharsets.UTF_8); Assert.assertEquals("Duplicate key: Both values should get masked", - "abc=XXXX&pqr=456XXXX&abc=XXXX", + "abc=XXXX&pqr=XXX1234&abc=XXXX", UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask)); //Duplicate key (not to be masked) with different values keyValueList = URLEncodedUtils .parse("abc=123&def=456&pqr=789&def=000&s=1", StandardCharsets.UTF_8); Assert.assertEquals("Duplicate key: no value should get masked", - "abc=XXXX&def=456&pqr=789XXXX&def=000&s=1", + "abc=XXXX&def=456&pqr=789&def=000&s=1", UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask)); //Empty param value keyValueList = URLEncodedUtils .parse("abc=123&def=&pqr=789&s=1", StandardCharsets.UTF_8); Assert.assertEquals("Incorrect url with empty query value", - "abc=XXXX&def=&pqr=789XXXX&s=1", + "abc=XXXX&def=&pqr=789&s=1", UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask)); //Empty param key @@ -111,14 +112,14 @@ public void testMaskUrlQueryParameters() throws Exception { .parse("abc=123&pqr=789&s=1", StandardCharsets.UTF_8); keyValueList.add(new BasicNameValuePair("null1", null)); Assert.assertEquals("Null value, incorrect query construction", - "abc=XXXX&pqr=789XXXX&s=1&null1=", + "abc=XXXX&pqr=789&s=1&null1=", UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask)); //Param (to be masked) with null value keyValueList.add(new BasicNameValuePair("null2", null)); fullMask.add("null2"); Assert.assertEquals("No mask should be added for null value", - "abc=XXXX&pqr=789XXXX&s=1&null1=&null2=", UriUtils + "abc=XXXX&pqr=789&s=1&null1=&null2=", UriUtils .maskUrlQueryParameters(keyValueList, fullMask, partialMask)); //no mask } From c69b3922cb447c0ab7184c912db410b77c0e0d53 Mon Sep 17 00:00:00 2001 From: sumangala Date: Fri, 23 Apr 2021 16:57:19 +0530 Subject: [PATCH 13/15] mask || server logs, revw --- .../azurebfs/services/AbfsHttpOperation.java | 18 +++++----- .../hadoop/fs/azurebfs/utils/UriUtils.java | 34 +++++++++++++------ ...ITestAzureBlobFileSystemDelegationSAS.java | 4 +-- .../services/TestAbfsHttpOperation.java | 10 +++--- .../fs/azurebfs/utils/TestUriUtils.java | 31 +++++++++++------ 5 files changed, 61 insertions(+), 36 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java index dd30752948d64..206316f9d6924 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java @@ -67,6 +67,8 @@ public class AbfsHttpOperation implements AbfsPerfLoggable { private int statusCode; private String statusDescription; private String storageErrorCode = ""; + private static final int DEFAULT_QUERY_STRINGBUILDER_CAPACITY = 550; + private static final int PARTIAL_MASK_VISIBLE_LEN = 4; private String storageErrorMessage = ""; private String clientRequestId = ""; private String requestId = ""; @@ -238,19 +240,19 @@ public String getMaskedUrl() { if (!shouldMask) { return url.toString(); } - if (this.maskedUrl != null) { - return this.maskedUrl; + if (maskedUrl != null) { + return maskedUrl; } - this.maskedUrl = UriUtils.getMaskedUrl(this.url); - return this.maskedUrl; + maskedUrl = UriUtils.getMaskedUrl(this.url); + return maskedUrl; } public String getMaskedEncodedUrl() { - if (this.maskedEncodedUrl != null) { - return this.maskedEncodedUrl; + if (maskedEncodedUrl != null) { + return maskedEncodedUrl; } - this.maskedEncodedUrl = UriUtils.encodedUrlStr(getMaskedUrl()); - return this.maskedEncodedUrl; + maskedEncodedUrl = UriUtils.encodedUrlStr(getMaskedUrl()); + return maskedEncodedUrl; } /** diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java index 8884c11510629..e27d54b443ca2 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java @@ -26,6 +26,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.regex.Pattern; import org.apache.commons.lang3.StringUtils; @@ -45,12 +46,14 @@ public final class UriUtils { private static final String ABFS_URI_REGEX = "[^.]+\\.dfs\\.(preprod\\.){0,1}core\\.windows\\.net"; private static final Pattern ABFS_URI_PATTERN = Pattern.compile(ABFS_URI_REGEX); - private static final HashSet FULL_MASK_PARAM_KEYS = new HashSet<>( + private static final Set FULL_MASK_PARAM_KEYS = new HashSet<>( Collections.singleton(QUERY_PARAM_SIGNATURE)); - private static final HashSet PARTIAL_MASK_PARAM_KEYS = new HashSet<>( + private static final Set PARTIAL_MASK_PARAM_KEYS = new HashSet<>( Arrays.asList(QUERY_PARAM_SKOID, QUERY_PARAM_SAOID, QUERY_PARAM_SUOID)); - private static final String FULL_MASK = "XXXX"; private static final Character CHAR_MASK = 'X'; + private static final String FULL_MASK = "XXXXX"; + private static final int DEFAULT_QUERY_STRINGBUILDER_CAPACITY = 550; + private static final int PARTIAL_MASK_VISIBLE_LEN = 18; /** * Checks whether a string includes abfs url. @@ -99,15 +102,25 @@ public static String generateUniqueTestPath() { } public static String maskUrlQueryParameters(List keyValueList, - HashSet queryParamsForFullMask, - HashSet queryParamsForPartialMask) { + Set queryParamsForFullMask, + Set queryParamsForPartialMask) { return maskUrlQueryParameters(keyValueList, queryParamsForFullMask, - queryParamsForPartialMask, 256); + queryParamsForPartialMask, DEFAULT_QUERY_STRINGBUILDER_CAPACITY); } + /** + * Generic function to mask a set of query parameters partially/fully and + * return the resultant query string + * @param keyValueList List of NameValuePair instances for query keys/values + * @param queryParamsForFullMask values for these params will appear as "XXXX" + * @param queryParamsForPartialMask values will be masked with 'X', except for + * the last PARTIAL_MASK_VISIBLE_LEN characters + * @param queryLen to initialize StringBuilder for the masked query + * @return the masked url query part + */ public static String maskUrlQueryParameters(List keyValueList, - HashSet queryParamsForFullMask, - HashSet queryParamsForPartialMask, int queryLen) { + Set queryParamsForFullMask, + Set queryParamsForPartialMask, int queryLen) { StringBuilder maskedUrl = new StringBuilder(queryLen); for (NameValuePair keyValuePair : keyValueList) { String key = keyValuePair.getName(); @@ -122,9 +135,10 @@ public static String maskUrlQueryParameters(List keyValueList, maskedUrl.append(FULL_MASK); } else if (queryParamsForPartialMask.contains(key)) { int valueLen = value.length(); - int maskedLen = valueLen - Math.min(4, valueLen); + int maskedLen = valueLen > PARTIAL_MASK_VISIBLE_LEN + ? PARTIAL_MASK_VISIBLE_LEN : valueLen / 2; + maskedUrl.append(value, 0, valueLen - maskedLen); maskedUrl.append(StringUtils.repeat(CHAR_MASK, maskedLen)); - maskedUrl.append(value, maskedLen, valueLen); } else { maskedUrl.append(value); } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java index 36b950c57bdb4..82773809dfadc 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java @@ -404,10 +404,10 @@ public void testSignatureMask() throws Exception { String encodedUrl = result.getMaskedEncodedUrl(); Assertions.assertThat(url.substring(url.indexOf("sig="))) .describedAs("Signature query param should be masked") - .startsWith("sig=XXXX"); + .startsWith("sig=XXXXX"); Assertions.assertThat(encodedUrl.substring(encodedUrl.indexOf("sig%3D"))) .describedAs("Signature query param should be masked") - .startsWith("sig%3DXXXX"); + .startsWith("sig%3DXXXXX"); } @Test diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java index 5e8fb9a08f5af..36914a4e4f365 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java @@ -34,24 +34,24 @@ public class TestAbfsHttpOperation { public void testMaskingAndEncoding() throws MalformedURLException, UnsupportedEncodingException { testIfMaskAndEncodeSuccessful("Where sig is the only query param", - "http://www.testurl.net?sig=abcd", "http://www.testurl.net?sig=XXXX"); + "http://www.testurl.net?sig=abcd", "http://www.testurl.net?sig=XXXXX"); testIfMaskAndEncodeSuccessful("Where oid is the only query param", "http://www.testurl.net?saoid=abcdef", - "http://www.testurl.net?saoid=XXcdef"); + "http://www.testurl.net?saoid=abcXXX"); testIfMaskAndEncodeSuccessful("Where sig is the first query param, oid is last", "http://www.testurl.net?sig=abcd&abc=xyz&saoid=pqrs456", - "http://www.testurl.net?sig=XXXX&abc=xyz&saoid=XXXs456"); + "http://www.testurl.net?sig=XXXXX&abc=xyz&saoid=pqrsXXX"); testIfMaskAndEncodeSuccessful( "Where sig/oid are neither first nor last query param", "http://www.testurl.net?lmn=abc&sig=abcd&suoid=mnop789&abc=xyz", - "http://www.testurl.net?lmn=abc&sig=XXXX&suoid=XXXp789&abc=xyz"); + "http://www.testurl.net?lmn=abc&sig=XXXXX&suoid=mnopXXX&abc=xyz"); testIfMaskAndEncodeSuccessful("Where sig is the last query param, oid is first", "http://www.testurl.net?skoid=pqrs123&abc=xyz&sig=abcd", - "http://www.testurl.net?skoid=XXXs123&abc=xyz&sig=XXXX"); + "http://www.testurl.net?skoid=pqrsXXX&abc=xyz&sig=XXXXX"); testIfMaskAndEncodeSuccessful("Where sig/oid query param are not present", "http://www.testurl.net?abc=xyz", "http://www.testurl.net?abc=xyz"); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java index 8b8925ca76e08..957516c0f6a9f 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java @@ -19,7 +19,6 @@ package org.apache.hadoop.fs.azurebfs.utils; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -59,6 +58,8 @@ public void testExtractRawAccountName() throws Exception { } @Test + // If a config for partial masking is introduced, this test will have to be + // modified for the config-controlled partial mask length public void testMaskUrlQueryParameters() throws Exception { HashSet fullMask = new HashSet<>(Arrays.asList("abc", "bcd")); HashSet partialMask = new HashSet<>( @@ -69,34 +70,42 @@ public void testMaskUrlQueryParameters() throws Exception { .parse("abc=123&pqr=45678&def=789&bcd=012&xyz=678", StandardCharsets.UTF_8); Assert.assertEquals("Incorrect masking", - "abc=XXXX&pqr=X5678&def=789&bcd=XXXX&xyz=678", + "abc=XXXXX&pqr=456XX&def=789&bcd=XXXXX&xyz=67X", + UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask)); + + //Mask GUIDs + keyValueList = URLEncodedUtils + .parse("abc=123&pqr=256877f2-c094-48c8-83df-ddb5825694fd&def=789", + StandardCharsets.UTF_8); + Assert.assertEquals("Incorrect partial masking for guid", + "abc=XXXXX&pqr=256877f2-c094-48c8XXXXXXXXXXXXXXXXXX&def=789", UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask)); //For params entered for both full and partial masks, full mask applies partialMask.add("abc"); Assert.assertEquals("Full mask should apply", - "abc=XXXX&pqr=X5678&def=789&bcd=XXXX&xyz=678", + "abc=XXXXX&pqr=256877f2-c094-48c8XXXXXXXXXXXXXXXXXX&def=789", UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask)); //Duplicate key (to be masked) with different values keyValueList = URLEncodedUtils .parse("abc=123&pqr=4561234&abc=789", StandardCharsets.UTF_8); Assert.assertEquals("Duplicate key: Both values should get masked", - "abc=XXXX&pqr=XXX1234&abc=XXXX", + "abc=XXXXX&pqr=4561XXX&abc=XXXXX", UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask)); //Duplicate key (not to be masked) with different values keyValueList = URLEncodedUtils - .parse("abc=123&def=456&pqr=789&def=000&s=1", StandardCharsets.UTF_8); - Assert.assertEquals("Duplicate key: no value should get masked", - "abc=XXXX&def=456&pqr=789&def=000&s=1", + .parse("abc=123&def=456&pqrs=789&def=000", StandardCharsets.UTF_8); + Assert.assertEquals("Duplicate key: Values should not get masked", + "abc=XXXXX&def=456&pqrs=789&def=000", UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask)); //Empty param value keyValueList = URLEncodedUtils .parse("abc=123&def=&pqr=789&s=1", StandardCharsets.UTF_8); Assert.assertEquals("Incorrect url with empty query value", - "abc=XXXX&def=&pqr=789&s=1", + "abc=XXXXX&def=&pqr=78X&s=1", UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask)); //Empty param key @@ -109,17 +118,17 @@ public void testMaskUrlQueryParameters() throws Exception { //Param (not to be masked) with null value keyValueList = URLEncodedUtils - .parse("abc=123&pqr=789&s=1", StandardCharsets.UTF_8); + .parse("abc=123&s=1", StandardCharsets.UTF_8); keyValueList.add(new BasicNameValuePair("null1", null)); Assert.assertEquals("Null value, incorrect query construction", - "abc=XXXX&pqr=789&s=1&null1=", + "abc=XXXXX&s=1&null1=", UriUtils.maskUrlQueryParameters(keyValueList, fullMask, partialMask)); //Param (to be masked) with null value keyValueList.add(new BasicNameValuePair("null2", null)); fullMask.add("null2"); Assert.assertEquals("No mask should be added for null value", - "abc=XXXX&pqr=789&s=1&null1=&null2=", UriUtils + "abc=XXXXX&s=1&null1=&null2=", UriUtils .maskUrlQueryParameters(keyValueList, fullMask, partialMask)); //no mask } From cb464328d51f1e2640d7e3eb2db2ece893f959ca Mon Sep 17 00:00:00 2001 From: sumangala Date: Fri, 23 Apr 2021 18:17:46 +0530 Subject: [PATCH 14/15] chkstyle --- .../apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java index 206316f9d6924..101392be22b1e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java @@ -67,8 +67,6 @@ public class AbfsHttpOperation implements AbfsPerfLoggable { private int statusCode; private String statusDescription; private String storageErrorCode = ""; - private static final int DEFAULT_QUERY_STRINGBUILDER_CAPACITY = 550; - private static final int PARTIAL_MASK_VISIBLE_LEN = 4; private String storageErrorMessage = ""; private String clientRequestId = ""; private String requestId = ""; From 6283044912e7b82381ede642d6c860547820a98f Mon Sep 17 00:00:00 2001 From: sumangala Date: Fri, 23 Apr 2021 18:24:26 +0530 Subject: [PATCH 15/15] chkstyle --- .../hadoop/fs/azurebfs/services/AbfsHttpOperation.java | 2 +- .../org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java index 101392be22b1e..f10097ded1efb 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java @@ -241,7 +241,7 @@ public String getMaskedUrl() { if (maskedUrl != null) { return maskedUrl; } - maskedUrl = UriUtils.getMaskedUrl(this.url); + maskedUrl = UriUtils.getMaskedUrl(url); return maskedUrl; } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java index 957516c0f6a9f..25d3f7caa4f39 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Set; import org.junit.Assert; import org.junit.Test; @@ -61,9 +62,8 @@ public void testExtractRawAccountName() throws Exception { // If a config for partial masking is introduced, this test will have to be // modified for the config-controlled partial mask length public void testMaskUrlQueryParameters() throws Exception { - HashSet fullMask = new HashSet<>(Arrays.asList("abc", "bcd")); - HashSet partialMask = new HashSet<>( - Arrays.asList("pqr", "xyz")); + Set fullMask = new HashSet<>(Arrays.asList("abc", "bcd")); + Set partialMask = new HashSet<>(Arrays.asList("pqr", "xyz")); //Partial and full masking test List keyValueList = URLEncodedUtils