HADOOP-17618. ABFS: Partially obfuscate SAS object IDs in Logs

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/AbfsRestOperationException.java

@@ -87,15 +87,15 @@ private static String formatMessage(final AbfsHttpOperation abfsHttpOperation) {
               "Operation failed: \"%1$s\", %2$s, HEAD, %3$s",
               abfsHttpOperation.getStatusDescription(),
               abfsHttpOperation.getStatusCode(),
-              abfsHttpOperation.getSignatureMaskedUrl());
+              abfsHttpOperation.getMaskedUrl());
     }
 
     return String.format(
             "Operation failed: \"%1$s\", %2$s, %3$s, %4$s, %5$s, \"%6$s\"",
             abfsHttpOperation.getStatusDescription(),
             abfsHttpOperation.getStatusCode(),
             abfsHttpOperation.getMethod(),
-            abfsHttpOperation.getSignatureMaskedUrl(),
+            abfsHttpOperation.getMaskedUrl(),
             abfsHttpOperation.getStorageErrorCode(),
             // Remove break line to ensure the request id and timestamp can be shown in console.
             abfsHttpOperation.getStorageErrorMessage().replaceAll("\\n", " "));

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java

@@ -21,16 +21,15 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.io.UnsupportedEncodingException;
 import java.net.HttpURLConnection;
 import java.net.URL;
-import java.net.URLEncoder;
 import java.util.List;
 import java.util.UUID;
 
 import javax.net.ssl.HttpsURLConnection;
 import javax.net.ssl.SSLSocketFactory;
 
+import org.apache.hadoop.fs.azurebfs.utils.UriUtils;
 import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
 import org.codehaus.jackson.JsonFactory;
 import org.codehaus.jackson.JsonParser;
@@ -51,8 +50,6 @@
 public class AbfsHttpOperation implements AbfsPerfLoggable {
   private static final Logger LOG = LoggerFactory.getLogger(AbfsHttpOperation.class);
 
-  public static final String SIGNATURE_QUERY_PARAM_KEY = "sig=";
-
   private static final int CONNECT_TIMEOUT = 30 * 1000;
   private static final int READ_TIMEOUT = 30 * 1000;
 
@@ -194,7 +191,7 @@ public String toString() {
     sb.append(",");
     sb.append(method);
     sb.append(",");
-    sb.append(getSignatureMaskedUrl());
+    sb.append(getMaskedUrl());
     return sb.toString();
   }
 
@@ -227,11 +224,26 @@ public String getLogString() {
       .append(" m=")
       .append(method)
       .append(" u=")
-      .append(getSignatureMaskedEncodedUrl());
+      .append(getMaskedEncodedUrl());
 
     return sb.toString();
   }
 
+  public String getMaskedUrl() {
+    if (maskedUrl != null) {
+      return maskedUrl;
+    }
+    maskedUrl = UriUtils.getMaskedUrl(url.toString());
+    return maskedUrl;
+  }
+
+  public String getMaskedEncodedUrl() {
+    if (maskedEncodedUrl != null) {
+      return maskedEncodedUrl;
+    }
+    return UriUtils.encodedUrlStr(getMaskedUrl());
+  }
+
   /**
    * Initializes a new HTTP request and opens the connection.
    *
@@ -521,43 +533,6 @@ private boolean isNullInputStream(InputStream stream) {
     return stream == null ? true : false;
   }
 
-  public static String getSignatureMaskedUrl(String url) {
-    int qpStrIdx = url.indexOf('?' + SIGNATURE_QUERY_PARAM_KEY);
-    if (qpStrIdx == -1) {
-      qpStrIdx = url.indexOf('&' + SIGNATURE_QUERY_PARAM_KEY);
-    }
-    if (qpStrIdx == -1) {
-      return url;
-    }
-    final int sigStartIdx = qpStrIdx + SIGNATURE_QUERY_PARAM_KEY.length() + 1;
-    final int ampIdx = url.indexOf("&", sigStartIdx);
-    final int sigEndIndex = (ampIdx != -1) ? ampIdx : url.length();
-    String signature = url.substring(sigStartIdx, sigEndIndex);
-    return url.replace(signature, "XXXX");
-  }
-
-  public static String encodedUrlStr(String url) {
-    try {
-      return URLEncoder.encode(url, "UTF-8");
-    } catch (UnsupportedEncodingException e) {
-      return "https%3A%2F%2Ffailed%2Fto%2Fencode%2Furl";
-    }
-  }
-
-  public String getSignatureMaskedUrl() {
-    if (this.maskedUrl == null) {
-      this.maskedUrl = getSignatureMaskedUrl(this.url.toString());
-    }
-    return this.maskedUrl;
-  }
-
-  public String getSignatureMaskedEncodedUrl() {
-    if (this.maskedEncodedUrl == null) {
-      this.maskedEncodedUrl = encodedUrlStr(getSignatureMaskedUrl());
-    }
-    return this.maskedEncodedUrl;
-  }
-
   public static class AbfsHttpOperationWithFixedResult extends AbfsHttpOperation {
     /**
      * Creates an instance to represent fixed results.

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java

@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.fs.azurebfs.utils;
 
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
 import java.util.regex.Pattern;
 
 /**
@@ -26,6 +28,9 @@
 public final class UriUtils {
   private static final String ABFS_URI_REGEX = "[^.]+\\.dfs\\.(preprod\\.){0,1}core\\.windows\\.net";
   private static final Pattern ABFS_URI_PATTERN = Pattern.compile(ABFS_URI_REGEX);
+  public static final String SIGNATURE_QUERY_PARAM_KEY = "sig=";
+  private static final String[] MASK_PARAM_KEYS = {"skoid=", "saoid=", "suoid=",
+      SIGNATURE_QUERY_PARAM_KEY};
 
   /**
    * Checks whether a string includes abfs url.
@@ -73,6 +78,36 @@ public static String generateUniqueTestPath() {
     return testUniqueForkId == null ? "/test" : "/" + testUniqueForkId + "/test";
   }
 
+  public static String encodedUrlStr(String url) {
+    try {
+      return URLEncoder.encode(url, "UTF-8");
+    } catch (UnsupportedEncodingException e) {
+      return "https%3A%2F%2Ffailed%2Fto%2Fencode%2Furl";
+    }
+  }
+
+  public static String getMaskedUrl(String url) {
+    for (String qpKey : MASK_PARAM_KEYS) {
+      int qpStrIdx = url.indexOf('&' + qpKey);
+      if (qpStrIdx == -1) {
+        qpStrIdx = url.indexOf('?' + qpKey);
+        if (qpStrIdx == -1) {
+          continue;
+        }
+      }
+      int startIdx = qpStrIdx + qpKey.length() + 1;
+      int ampIdx = url.indexOf("&", startIdx);
+      int endIndex = (ampIdx != -1) ? ampIdx : url.length();
+      if (qpKey.equals(SIGNATURE_QUERY_PARAM_KEY)) {
+        String signature = url.substring(startIdx, endIndex);
+        url = url.replace(signature, "XXXX");
+      } else {
+        url = url.substring(0, startIdx + 4) + "XXXX" + url.substring(endIndex);
+      }
+    }
+    return url;
+  }
+
   private UriUtils() {
   }
 }

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java

@@ -64,6 +64,7 @@
  */
 public class ITestAzureBlobFileSystemDelegationSAS extends AbstractAbfsIntegrationTest {
   private static final String TEST_GROUP = UUID.randomUUID().toString();
+  private static final String[] OID_KEYS = {"skoid=", "saoid=", "suoid="};
 
   private static final Logger LOG =
       LoggerFactory.getLogger(ITestAzureBlobFileSystemDelegationSAS.class);
@@ -400,8 +401,8 @@ public void testSignatureMask() throws Exception {
     AbfsRestOperation abfsHttpRestOperation = fs.getAbfsClient()
         .renamePath(src, "/testABC" + "/abc.txt", null);
     AbfsHttpOperation result = abfsHttpRestOperation.getResult();
-    String url = result.getSignatureMaskedUrl();
-    String encodedUrl = result.getSignatureMaskedEncodedUrl();
+    String url = result.getMaskedUrl();
+    String encodedUrl = result.getMaskedEncodedUrl();
     Assertions.assertThat(url.substring(url.indexOf("sig=")))
         .describedAs("Signature query param should be masked")
         .startsWith("sig=XXXX");
@@ -410,6 +411,31 @@ public void testSignatureMask() throws Exception {
         .startsWith("sig%3DXXXX");
   }
 
+  @Test
+  public void testSASObjectIDMask() throws IOException {
+    final AzureBlobFileSystem fs = getFileSystem();
+    AbfsHttpOperation abfsHttpOperation = fs.getAbfsClient().getAclStatus("/")
+        .getResult();
+    String url = abfsHttpOperation.getMaskedUrl();
+    String encodedUrl = abfsHttpOperation.getMaskedEncodedUrl();
+    int startIndex;
+    for (String qpKey : OID_KEYS) {
+      startIndex = url.indexOf(qpKey);
+      if (startIndex != -1) {
+        Assertions.assertThat(url.substring(startIndex + qpKey.length() + 4))
+            .describedAs("OID query param should be masked in url")
+            .startsWith("XXXX");
+      }
+      startIndex = encodedUrl.indexOf(qpKey);
+      if (startIndex != -1) {
+        Assertions
+            .assertThat(encodedUrl.substring(startIndex + qpKey.length() + 6))
+            .describedAs("OID query param should be masked in encoded url")
+            .startsWith("XXXX");
+      }
+    }
+  }
+
   @Test
   public void testSignatureMaskOnExceptionMessage() throws Exception {
     intercept(IOException.class, "sig=XXXX",

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java

@@ -25,6 +25,8 @@
 import org.assertj.core.api.Assertions;
 import org.junit.Test;
 
+import org.apache.hadoop.fs.azurebfs.utils.UriUtils;
+
 public class TestAbfsHttpOperation {
 
   @Test
@@ -33,59 +35,63 @@ public void testMaskingAndEncoding()
     testIfMaskAndEncodeSuccessful("Where sig is the only query param",
         "http://www.testurl.net?sig=abcd", "http://www.testurl.net?sig=XXXX");
 
-    testIfMaskAndEncodeSuccessful("Where sig is the first query param",
-        "http://www.testurl.net?sig=abcd&abc=xyz",
-        "http://www.testurl.net?sig=XXXX&abc=xyz");
+    testIfMaskAndEncodeSuccessful("Where oid is the only query param",
+        "http://www.testurl.net?saoid=abcdef",
+        "http://www.testurl.net?saoid=abcdXXXX");
+
+    testIfMaskAndEncodeSuccessful("Where sig is the first query param, oid is last",
+        "http://www.testurl.net?sig=abcd&abc=xyz&saoid=pqrs456",
+        "http://www.testurl.net?sig=XXXX&abc=xyz&saoid=pqrsXXXX");
 
     testIfMaskAndEncodeSuccessful(
-        "Where sig is neither first nor last query param",
-        "http://www.testurl.net?lmn=abc&sig=abcd&abc=xyz",
-        "http://www.testurl.net?lmn=abc&sig=XXXX&abc=xyz");
+        "Where sig/oid are neither first nor last query param",
+        "http://www.testurl.net?lmn=abc&sig=abcd&suoid=mnop789&abc=xyz",
+        "http://www.testurl.net?lmn=abc&sig=XXXX&suoid=mnopXXXX&abc=xyz");
 
-    testIfMaskAndEncodeSuccessful("Where sig is the last query param",
-        "http://www.testurl.net?abc=xyz&sig=abcd",
-        "http://www.testurl.net?abc=xyz&sig=XXXX");
+    testIfMaskAndEncodeSuccessful("Where sig is the last query param, oid is first",
+        "http://www.testurl.net?skoid=pqrs123&abc=xyz&sig=abcd",
+        "http://www.testurl.net?skoid=pqrsXXXX&abc=xyz&sig=XXXX");
 
-    testIfMaskAndEncodeSuccessful("Where sig query param is not present",
+    testIfMaskAndEncodeSuccessful("Where sig/oid query param are not present",
         "http://www.testurl.net?abc=xyz", "http://www.testurl.net?abc=xyz");
 
     testIfMaskAndEncodeSuccessful(
-        "Where sig query param is not present but mysig",
-        "http://www.testurl.net?abc=xyz&mysig=qwerty",
-        "http://www.testurl.net?abc=xyz&mysig=qwerty");
+        "Where sig/oid query param are not present but mysig and myoid",
+        "http://www.testurl.net?abc=xyz&mysig=qwerty&mysaoid=uvw",
+        "http://www.testurl.net?abc=xyz&mysig=qwerty&mysaoid=uvw");
 
     testIfMaskAndEncodeSuccessful(
-        "Where sig query param is not present but sigmy",
-        "http://www.testurl.net?abc=xyz&sigmy=qwerty",
-        "http://www.testurl.net?abc=xyz&sigmy=qwerty");
+        "Where sig/oid query param is not present but sigmy and oidmy",
+        "http://www.testurl.net?abc=xyz&sigmy=qwerty&skoidmy=uvw",
+        "http://www.testurl.net?abc=xyz&sigmy=qwerty&skoidmy=uvw");
 
     testIfMaskAndEncodeSuccessful(
-        "Where sig query param is not present but a " + "value sig",
-        "http://www.testurl.net?abc=xyz&mnop=sig",
-        "http://www.testurl.net?abc=xyz&mnop=sig");
+        "Where sig/oid query param is not present but values sig and oid",
+        "http://www.testurl.net?abc=xyz&mnop=sig&pqr=saoid",
+        "http://www.testurl.net?abc=xyz&mnop=sig&pqr=saoid");
 
     testIfMaskAndEncodeSuccessful(
-        "Where sig query param is not present but a " + "value ends with sig",
-        "http://www.testurl.net?abc=xyz&mnop=abcsig",
-        "http://www.testurl.net?abc=xyz&mnop=abcsig");
+        "Where sig/oid query param is not present but a value ends with sig/oid",
+        "http://www.testurl.net?abc=xyzsaoid&mnop=abcsig",
+        "http://www.testurl.net?abc=xyzsaoid&mnop=abcsig");
 
     testIfMaskAndEncodeSuccessful(
-        "Where sig query param is not present but a " + "value starts with sig",
-        "http://www.testurl.net?abc=xyz&mnop=sigabc",
-        "http://www.testurl.net?abc=xyz&mnop=sigabc");
+        "Where sig/oid query param is not present but a value starts with sig/oid",
+        "http://www.testurl.net?abc=saoidxyz&mnop=sigabc",
+        "http://www.testurl.net?abc=saoidxyz&mnop=sigabc");
   }
 
   private void testIfMaskAndEncodeSuccessful(final String scenario,
       final String url, final String expectedMaskedUrl)
       throws UnsupportedEncodingException {
 
-    Assertions.assertThat(AbfsHttpOperation.getSignatureMaskedUrl(url))
+    Assertions.assertThat(UriUtils.getMaskedUrl(url))
         .describedAs(url + " (" + scenario + ") after masking should be: "
             + expectedMaskedUrl).isEqualTo(expectedMaskedUrl);
 
     final String expectedMaskedEncodedUrl = URLEncoder
         .encode(expectedMaskedUrl, "UTF-8");
-    Assertions.assertThat(AbfsHttpOperation.encodedUrlStr(expectedMaskedUrl))
+    Assertions.assertThat(UriUtils.encodedUrlStr(expectedMaskedUrl))
         .describedAs(
             url + " (" + scenario + ") after masking and encoding should "
                 + "be: " + expectedMaskedEncodedUrl)