diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java index 3b15222ff47ff3..541f249658757a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java @@ -176,6 +176,10 @@ public final class AbfsHttpConstants { public static final String COPY_PROGRESS = "CopyProgress"; public static final String COPY_COMPLETION_TIME = "CopyCompletionTime"; public static final String COPY_STATUS_DESCRIPTION = "CopyStatusDescription"; + public static final String BLOB_ERROR_CODE_START_XML = ""; + public static final String BLOB_ERROR_CODE_END_XML = ""; + public static final String BLOB_ERROR_MESSAGE_START_XML = ""; + public static final String BLOB_ERROR_MESSAGE_END_XML = ""; private AbfsHttpConstants() {} } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java index 62e7902108fcf6..2d38fa29845ccb 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java @@ -18,13 +18,12 @@ package org.apache.hadoop.fs.azurebfs.services; -import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -37,6 +36,7 @@ import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; +import org.apache.commons.io.IOUtils; import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.fs.azurebfs.utils.UriUtils; import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory; @@ -60,9 +60,14 @@ import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.BLOCKLIST; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.BLOCK_NAME; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.BLOB_ERROR_CODE_END_XML; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.BLOB_ERROR_CODE_START_XML; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.COMMITTED_BLOCKS; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EQUAL; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.BLOB_ERROR_MESSAGE_END_XML; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.BLOB_ERROR_MESSAGE_START_XML; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.NAME; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.WASB_DNS_PREFIX; import static org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams.QUERY_PARAM_COMP; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.COMP_LIST; @@ -82,7 +87,6 @@ public class AbfsHttpOperation implements AbfsPerfLoggable { private static final int ONE_THOUSAND = 1000; private static final int ONE_MILLION = ONE_THOUSAND * ONE_THOUSAND; - private final String method; private final URL url; private String maskedUrl; @@ -466,7 +470,7 @@ public void processResponse(final byte[] buffer, final int offset, final int len } if (statusCode >= HttpURLConnection.HTTP_BAD_REQUEST) { - processStorageErrorResponse(); + processServerErrorResponse(); if (this.isTraceEnabled) { this.recvResponseTimeMs += elapsedTimeMs(startTime); } @@ -527,6 +531,15 @@ public void processResponse(final byte[] buffer, final int offset, final int len } } + @VisibleForTesting + void processServerErrorResponse() throws IOException { + if (getBaseUrl().contains(WASB_DNS_PREFIX)) { + processBlobStorageErrorResponse(); + } else { + processDfsStorageErrorResponse(); + } + } + /** * Parse the stream from the response and set {@link #blobList} field of this * class. @@ -638,8 +651,8 @@ private HttpURLConnection openConnection() throws IOException { * } * */ - private void processStorageErrorResponse() { - try (InputStream stream = connection.getErrorStream()) { + private void processDfsStorageErrorResponse() { + try (InputStream stream = getConnectionErrorStream()) { if (stream == null) { return; } @@ -680,6 +693,48 @@ private void processStorageErrorResponse() { } } + /** + * Extract errorCode and errorMessage from errorStream populated by server. + * Error-message in the form of: + *
+   *   {@code
+   *   
+   *   
+   *      string-value
+   *      string-value
+   *   
+   * }
+   * 
+ * + * Reference + */ + private void processBlobStorageErrorResponse() throws IOException { + InputStream errorStream = getConnectionErrorStream(); + if (errorStream == null) { + return; + } + final String data = IOUtils.toString(errorStream, StandardCharsets.UTF_8); + + int codeStartFirstInstance = data.indexOf(BLOB_ERROR_CODE_START_XML); + int codeEndFirstInstance = data.indexOf(BLOB_ERROR_CODE_END_XML); + if (codeEndFirstInstance != -1 && codeStartFirstInstance != -1) { + storageErrorCode = data.substring(codeStartFirstInstance, + codeEndFirstInstance).replace(BLOB_ERROR_CODE_START_XML, ""); + } + + int msgStartFirstInstance = data.indexOf(BLOB_ERROR_MESSAGE_START_XML); + int msgEndFirstInstance = data.indexOf(BLOB_ERROR_MESSAGE_END_XML); + if (msgEndFirstInstance != -1 && msgStartFirstInstance != -1) { + storageErrorMessage = data.substring(msgStartFirstInstance, + msgEndFirstInstance).replace(BLOB_ERROR_MESSAGE_START_XML, ""); + } + } + + @VisibleForTesting + InputStream getConnectionErrorStream() { + return connection.getErrorStream(); + } + /** * Returns the elapsed time in milliseconds. */ diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java index 36914a4e4f3658..6806cd955e86e5 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsHttpOperation.java @@ -18,16 +18,23 @@ package org.apache.hadoop.fs.azurebfs.services; +import java.io.ByteArrayInputStream; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import org.assertj.core.api.Assertions; import org.junit.Test; +import org.mockito.Mockito; import org.apache.hadoop.fs.azurebfs.utils.UriUtils; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_METHOD_POST; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_METHOD_PUT; + public class TestAbfsHttpOperation { @Test @@ -96,6 +103,77 @@ public void testUrlWithNullValues() "http://www.testurl.net?abc=xyz&pqr=&mnop="); } + @Test + public void testParseStorageErrorStreamBlob() throws Exception { + AbfsHttpOperation op = Mockito.spy(new AbfsHttpOperation( + new URL("https://account.blob.core.windows.net/container/path"), + HTTP_METHOD_PUT, new ArrayList<>())); + String xmlString = "\n" + + "\n" + + " string-value-code\n" + + " string-value-message\n" + + ""; + ByteArrayInputStream inputStream = new ByteArrayInputStream( + xmlString.getBytes( + StandardCharsets.UTF_8)); + Mockito.doReturn(inputStream).when(op).getConnectionErrorStream(); + op.processServerErrorResponse(); + Assertions.assertThat(op.getStorageErrorCode()) + .isEqualTo("string-value-code"); + Assertions.assertThat(op.getStorageErrorMessage()) + .isEqualTo("string-value-message"); + + op = Mockito.spy(new AbfsHttpOperation( + new URL("https://account.blob.core.windows.net/container/path"), + HTTP_METHOD_PUT, new ArrayList<>())); + xmlString = "\n" + + "\n" + + " string-value-code\n" + + " string-value-message\n" + + ""; + inputStream = new ByteArrayInputStream(xmlString.getBytes( + StandardCharsets.UTF_8)); + Mockito.doReturn(inputStream).when(op).getConnectionErrorStream(); + op.processServerErrorResponse(); + Assertions.assertThat(op.getStorageErrorCode()).isEmpty(); + Assertions.assertThat(op.getStorageErrorMessage()) + .isEqualTo("string-value-message"); + + op = Mockito.spy(new AbfsHttpOperation( + new URL("https://account.blob.core.windows.net/container/path"), + HTTP_METHOD_PUT, new ArrayList<>())); + xmlString = "\n" + + "\n" + + " string-value-code\n" + + " string-value-message\n" + + ""; + inputStream = new ByteArrayInputStream(xmlString.getBytes( + StandardCharsets.UTF_8)); + Mockito.doReturn(inputStream).when(op).getConnectionErrorStream(); + op.processServerErrorResponse(); + Assertions.assertThat(op.getStorageErrorCode()) + .isEqualTo("string-value-code"); + Assertions.assertThat(op.getStorageErrorMessage()).isEmpty(); + } + + @Test + public void testParseStorageErrorStreamDfs() throws Exception { + AbfsHttpOperation op = Mockito.spy(new AbfsHttpOperation( + new URL("https://account.dfs.core.windows.net/container/path"), + HTTP_METHOD_PUT, new ArrayList<>())); + String xmlString + = "{\"error\":{\"code\":\"errorCode\", \"message\":\"errorMessage\"}}"; + ByteArrayInputStream inputStream = new ByteArrayInputStream( + xmlString.getBytes( + StandardCharsets.UTF_8)); + Mockito.doReturn(inputStream).when(op).getConnectionErrorStream(); + op.processServerErrorResponse(); + Assertions.assertThat(op.getStorageErrorCode()) + .isEqualTo("errorCode"); + Assertions.assertThat(op.getStorageErrorMessage()) + .isEqualTo("errorMessage"); + } + private void testIfMaskAndEncodeSuccessful(final String scenario, final String url, final String expectedMaskedUrl) throws UnsupportedEncodingException, MalformedURLException {