getMetadata() {
+ return metadata;
+ }
+
+ public AzureBlobFileSystemException getFailureException() {
+ return ex;
+ }
+
+ public Path getBlobDstPath(Path dstBlobPath) {
+ return null;
+ }
+
+ public void setName(final String name) {
+ this.name = name;
+ }
+
+ public void setPath(final Path path) {
+ this.path = path;
+ }
+
+ public void setUrl(final String url) {
+ this.url = url;
+ }
+
+ public void setIsDirectory(final Boolean isDirectory) {
+ this.isDirectory = isDirectory;
+ }
+
+ public void setETag(final String eTag) {
+ this.eTag = eTag;
+ }
+
+ public void setLastModifiedTime(final long lastModifiedTime) {
+ this.lastModifiedTime = lastModifiedTime;
+ }
+
+ public void setCreationTime(final long creationTime) {
+ this.creationTime = creationTime;
+ }
+
+ public void setOwner(final String owner) {
+ this.owner = owner;
+ }
+
+ public void setGroup(final String group) {
+ this.group = group;
+ }
+
+ public void setPermission(final String permission) {
+ this.permission = permission;
+ }
+
+ public void setAcl(final String acl) {
+ this.acl = acl;
+ }
+
+ public void setContentLength(final Long contentLength) {
+ this.contentLength = contentLength;
+ }
+
+ public void setCopyId(final String copyId) {
+ this.copyId = copyId;
+ }
+
+ public void setCopyStatus(final String copyStatus) {
+ this.copyStatus = copyStatus;
+ }
+
+ public void setCopySourceUrl(final String copySourceUrl) {
+ this.copySourceUrl = copySourceUrl;
+ }
+
+ public void setCopyProgress(final String copyProgress) {
+ this.copyProgress = copyProgress;
+ }
+
+ public void setCopyStatusDescription(final String copyStatusDescription) {
+ this.copyStatusDescription = copyStatusDescription;
+ }
+
+ public void setCopyCompletionTime(final long copyCompletionTime) {
+ this.copyCompletionTime = copyCompletionTime;
+ }
+
+ public void addMetadata(String key, String value) {
+ metadata.put(key, value);
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/BlockStatus.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/BlockStatus.java
new file mode 100644
index 00000000000000..387f09f4ecc1ac
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/BlockStatus.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+public enum BlockStatus {
+ NEW,
+ SUCCESS,
+ FAILED
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java
index 89d99471a82145..dee1d374d4a046 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java
@@ -24,6 +24,8 @@
import org.apache.hadoop.fs.azurebfs.AbfsConfiguration;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_CONTINUE;
+
/**
* Retry policy used by AbfsClient.
* */
@@ -56,6 +58,13 @@ public class ExponentialRetryPolicy {
*/
private static final double MAX_RANDOM_RATIO = 1.2;
+ /**
+ * Qualifies for retry based on
+ * https://learn.microsoft.com/en-us/azure/active-directory/
+ * managed-identities-azure-resources/how-to-use-vm-token#error-handling
+ */
+ private static final int HTTP_TOO_MANY_REQUESTS = 429;
+
/**
* Holds the random number generator used to calculate randomized backoff intervals
*/
@@ -118,7 +127,12 @@ public ExponentialRetryPolicy(final int retryCount, final int minBackoff, final
/**
* Returns if a request should be retried based on the retry count, current response,
- * and the current strategy.
+ * and the current strategy. The valid http status code lies in the range of 1xx-5xx.
+ * But an invalid status code might be set due to network or timeout kind of issues.
+ * Such invalid status code also qualify for retry.
+ * HTTP status code 410 qualifies for retry based on
+ * https://docs.microsoft.com/en-in/azure/virtual-machines/linux/
+ * instance-metadata-service?tabs=windows#errors-and-debugging
*
* @param retryCount The current retry attempt count.
* @param statusCode The status code of the response, or -1 for socket error.
@@ -126,8 +140,10 @@ public ExponentialRetryPolicy(final int retryCount, final int minBackoff, final
*/
public boolean shouldRetry(final int retryCount, final int statusCode) {
return retryCount < this.retryCount
- && (statusCode == -1
+ && (statusCode < HTTP_CONTINUE
|| statusCode == HttpURLConnection.HTTP_CLIENT_TIMEOUT
+ || statusCode == HttpURLConnection.HTTP_GONE
+ || statusCode == HTTP_TOO_MANY_REQUESTS
|| (statusCode >= HttpURLConnection.HTTP_INTERNAL_ERROR
&& statusCode != HttpURLConnection.HTTP_NOT_IMPLEMENTED
&& statusCode != HttpURLConnection.HTTP_VERSION));
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ListBlobConsumer.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ListBlobConsumer.java
new file mode 100644
index 00000000000000..bf6981505e4ead
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ListBlobConsumer.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+import java.util.List;
+
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
+
+public class ListBlobConsumer {
+
+ private final ListBlobQueue listBlobQueue;
+
+ public ListBlobConsumer(final ListBlobQueue listBlobQueue) {
+ this.listBlobQueue = listBlobQueue;
+ }
+
+ public List consume() throws AzureBlobFileSystemException {
+ if (listBlobQueue.getException() != null) {
+ throw listBlobQueue.getException();
+ }
+ return listBlobQueue.dequeue();
+ }
+
+ public Boolean isCompleted() {
+ return listBlobQueue.getIsCompleted()
+ && listBlobQueue.size() == 0;
+ }
+
+ /**
+ * Register consumer failure.
+ */
+ public void fail() {
+ listBlobQueue.consumptionFailed();
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ListBlobProducer.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ListBlobProducer.java
new file mode 100644
index 00000000000000..a198dc706c84a0
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ListBlobProducer.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
+import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
+
+/**
+ * ListBlob API can give maximum of 5000 blobs. If there are (~n*5000) blobs, the
+ * client would need to call the listBlob API n times. This would have two consequences:
+ *
+ * -
+ * The consumer of the result of lists of blob would have to wait until all
+ * the blobs are received. The consumer could have used the time to start
+ * processing the blobs already in memory. The wait for receiving all the blobs
+ * would lead the processing more time. Lets say consumer need m time-units to process
+ * one blob. Lets assume that each set of blobs have x blobs. In total there
+ * have to be n sets Lets say that client needs t time to get all the blobs. If consumer
+ * wait for all the blobs to be received, the total time taken would be:
+ *
t + (n * x * m)
+ * Now, lets assume that consumer in parallel work on the available the blobs,
+ * time taken would be:
+ * t + (((n * x) - t/m) * m)
+ *
+ * -
+ * Since the information of the blobs have to be maintained in memory until the
+ * computation on the list is done. On low configuration machine, it may lead
+ * to OOM.
+ *
+ *
+ * In this design, the producer on a parallel thread to the main thread, will call
+ * ListBlob API and will populate {@link ListBlobQueue}, which would be dequeued by
+ * the main thread which will run the computation on the available blobs.
+ *
+ * How its different from {@link AbfsListStatusRemoteIterator}?
+ * It provides an iterator which on {@link AbfsListStatusRemoteIterator#hasNext()} checks
+ * if there are blobs available in memory. If not it will call Listing API on server for
+ * the next set of blobs. But here, it make the process sequential. As in, when the processing
+ * on whole set of blobs available in memory are done, after that only next set of blobs are
+ * fetched.
+ */
+public class ListBlobProducer {
+
+ private final AbfsClient client;
+
+ private final ListBlobQueue listBlobQueue;
+
+ private final String src;
+
+ private final TracingContext tracingContext;
+
+ private String nextMarker;
+ private final Thread thread;
+
+ public ListBlobProducer(final String src,
+ final AbfsClient abfsClient,
+ final ListBlobQueue listBlobQueue,
+ final String initNextMarker,
+ TracingContext tracingContext) {
+ this.src = src;
+ this.client = abfsClient;
+ this.tracingContext = tracingContext;
+ this.listBlobQueue = listBlobQueue;
+ listBlobQueue.setProducer(this);
+ this.nextMarker = initNextMarker;
+ thread = new Thread(() -> {
+ do {
+ int maxResult = listBlobQueue.availableSize();
+ if (maxResult == 0) {
+ continue;
+ }
+ AbfsRestOperation op = null;
+ try {
+ op = client.getListBlobs(nextMarker, src, null, maxResult, tracingContext);
+ } catch (AzureBlobFileSystemException ex) {
+ listBlobQueue.setFailed(ex);
+ return;
+ }
+ BlobList blobList = op.getResult().getBlobList();
+ nextMarker = blobList.getNextMarker();
+ listBlobQueue.enqueue(blobList.getBlobPropertyList());
+ if (nextMarker == null) {
+ listBlobQueue.complete();
+ }
+ } while(nextMarker != null && !listBlobQueue.getConsumptionFailed());
+ });
+ thread.start();
+ }
+
+ @VisibleForTesting
+ public void waitForProcessCompletion() throws InterruptedException {
+ thread.join();
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ListBlobQueue.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ListBlobQueue.java
new file mode 100644
index 00000000000000..3de0c85ec50006
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ListBlobQueue.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Queue;
+
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
+
+public class ListBlobQueue {
+
+ private final Queue blobLists;
+
+ private int totalProduced = 0;
+
+ private int totalConsumed = 0;
+
+ private Boolean isCompleted = false;
+ private Boolean isConsumptionFailed = false;
+
+ private AzureBlobFileSystemException failureFromProducer;
+
+ /**
+ * Since, Producer just spawns a thread and there are no public method for the
+ * class. Keeping its address in this object will prevent accidental GC close
+ * on the producer object.
+ */
+ private ListBlobProducer producer;
+
+ private final int maxSize;
+ private final int maxConsumedBlobCount;
+
+ /**
+ * @param maxSize maxSize of the queue.
+ * @param maxConsumedBlobCount maximum number of blobs that would be returned
+ * by {@link #dequeue()} method.
+ */
+ public ListBlobQueue(int maxSize, int maxConsumedBlobCount) {
+ blobLists = new ArrayDeque<>(maxSize);
+ this.maxSize = maxSize;
+ this.maxConsumedBlobCount = maxConsumedBlobCount;
+ }
+
+ /**
+ * @param initBlobList list of blobProperties to be enqueued in th queue
+ * @param maxSize maxSize of the queue.
+ * @param maxConsumedBlobCount maximum number of blobs that would be returned
+ * by {@link #dequeue()} method.
+ */
+ public ListBlobQueue(List initBlobList, int maxSize, int maxConsumedBlobCount) {
+ this(maxSize, maxConsumedBlobCount);
+ if (initBlobList != null) {
+ enqueue(initBlobList);
+ }
+ }
+
+ void setProducer(ListBlobProducer producer) {
+ if (this.producer == null) {
+ this.producer = producer;
+ }
+ }
+
+ void setFailed(AzureBlobFileSystemException failure) {
+ failureFromProducer = failure;
+ }
+
+ public void complete() {
+ isCompleted = true;
+ }
+
+ void consumptionFailed() {
+ isConsumptionFailed = true;
+ }
+
+ Boolean getConsumptionFailed() {
+ return isConsumptionFailed;
+ }
+
+ public Boolean getIsCompleted() {
+ return isCompleted;
+ }
+
+ AzureBlobFileSystemException getException() {
+ return failureFromProducer;
+ }
+
+ public void enqueue(List blobProperties) {
+ blobLists.addAll(blobProperties);
+ }
+
+ public List dequeue() {
+ List blobProperties = new ArrayList<>();
+ int counter = 0;
+ while (counter < maxConsumedBlobCount && blobLists.size() > 0) {
+ blobProperties.add(blobLists.poll());
+ counter++;
+ }
+ return blobProperties;
+ }
+
+ public int size() {
+ return blobLists.size();
+ }
+
+ public int availableSize() {
+ return maxSize - blobLists.size();
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/OperativeEndpoint.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/OperativeEndpoint.java
new file mode 100644
index 00000000000000..93667eb77df84b
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/OperativeEndpoint.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.azurebfs.services;
+
+import org.apache.hadoop.fs.azurebfs.AbfsConfiguration;
+
+/**
+ * This class is mainly to unify the fallback for all API's to DFS endpoint at a single spot.
+ */
+public class OperativeEndpoint {
+ public static boolean isMkdirEnabledOnDFS(AbfsConfiguration abfsConfiguration) {
+ if (abfsConfiguration.getPrefixMode() == PrefixMode.BLOB) {
+ return abfsConfiguration.shouldMkdirFallbackToDfs();
+ } else {
+ return true;
+ }
+ }
+
+ public static boolean isIngressEnabledOnDFS(PrefixMode prefixMode, AbfsConfiguration abfsConfiguration) {
+ if (prefixMode == PrefixMode.BLOB) {
+ return abfsConfiguration.shouldIngressFallbackToDfs();
+ } else {
+ return true;
+ }
+ }
+
+ public static boolean isReadEnabledOnDFS(AbfsConfiguration abfsConfiguration) {
+ if (abfsConfiguration.getPrefixMode() == PrefixMode.BLOB) {
+ return abfsConfiguration.shouldReadFallbackToDfs();
+ }
+ return true;
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/PathInformation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/PathInformation.java
new file mode 100644
index 00000000000000..e4f2790565fbff
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/PathInformation.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+public class PathInformation {
+ private Boolean pathExists;
+ private Boolean isDirectory;
+
+ public PathInformation(Boolean pathExists, Boolean isDirectory) {
+ this.pathExists = pathExists;
+ this.isDirectory = isDirectory;
+ }
+
+ public Boolean getPathExists() {
+ return pathExists;
+ }
+
+ public Boolean getIsDirectory() {
+ return isDirectory;
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/PrefixMode.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/PrefixMode.java
new file mode 100644
index 00000000000000..853a22f4f21c9e
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/PrefixMode.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+public enum PrefixMode {
+ DFS,
+ BLOB
+}
+
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java
index 4eefb9fdf2c7e3..0f91afe0982dbf 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java
@@ -101,6 +101,7 @@ private void init() {
// hide instance constructor
private ReadBufferManager() {
+ LOGGER.trace("Creating readbuffer manager with HADOOP-18546 patch");
}
@@ -544,7 +545,6 @@ public synchronized void purgeBuffersForStream(AbfsInputStream stream) {
LOGGER.debug("Purging stale buffers for AbfsInputStream {} ", stream);
readAheadQueue.removeIf(readBuffer -> readBuffer.getStream() == stream);
purgeList(stream, completedReadList);
- purgeList(stream, inProgressList);
}
/**
@@ -642,4 +642,9 @@ void testMimicFullUseAndAddFailedBuffer(ReadBuffer buf) {
freeList.clear();
completedReadList.add(buf);
}
+
+ @VisibleForTesting
+ int getNumBuffers() {
+ return NUM_BUFFERS;
+ }
}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/RenameAtomicityUtils.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/RenameAtomicityUtils.java
new file mode 100644
index 00000000000000..7490d3c090a4b6
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/RenameAtomicityUtils.java
@@ -0,0 +1,399 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.TimeZone;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
+import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
+
+/**
+ * For a directory enabled for atomic-rename, before rename starts, a
+ * file with -RenamePending.json suffix is created. In this file, the states required
+ * for the rename are given. This file is created by {@link #preRename(Boolean, String)} ()} method.
+ * This is important in case the JVM process crashes during rename, the atomicity
+ * will be maintained, when the job calls {@link AzureBlobFileSystem#listStatus(Path)}
+ * or {@link AzureBlobFileSystem#getFileStatus(Path)}. On these API calls to filesystem,
+ * it will be checked if there is any RenamePending JSON file. If yes, the rename
+ * would be resumed as per the file.
+ */
+public class RenameAtomicityUtils {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ RenameAtomicityUtils.class);
+
+ private final AzureBlobFileSystem azureBlobFileSystem;
+ private Path srcPath;
+ private Path dstPath;
+ private TracingContext tracingContext;
+ private Boolean isReDone;
+
+ private static final int MAX_RENAME_PENDING_FILE_SIZE = 10000000;
+ private static final int FORMATTING_BUFFER = 10000;
+
+ public static final String SUFFIX = "-RenamePending.json";
+
+ private static final ObjectReader READER = new ObjectMapper()
+ .configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true)
+ .readerFor(JsonNode.class);
+
+ public RenameAtomicityUtils(final AzureBlobFileSystem azureBlobFileSystem,
+ final Path srcPath,
+ final Path dstPath,
+ final TracingContext tracingContext) throws IOException {
+ this.azureBlobFileSystem = azureBlobFileSystem;
+ this.srcPath = srcPath;
+ this.dstPath = dstPath;
+ this.tracingContext = tracingContext;
+ }
+
+ public RenameAtomicityUtils(final AzureBlobFileSystem azureBlobFileSystem,
+ final Path renamePendingJsonPath,
+ final RedoRenameInvocation redoRenameInvocation,
+ final String srcEtag,
+ final AbfsInputStream renamePendingJsonInputStream)
+ throws IOException {
+ this.azureBlobFileSystem = azureBlobFileSystem;
+ final RenamePendingFileInfo renamePendingFileInfo = readFile(
+ renamePendingJsonPath, renamePendingJsonInputStream);
+ if (renamePendingFileInfo != null
+ && renamePendingFileInfo.eTag.equalsIgnoreCase(srcEtag)) {
+ redoRenameInvocation.redo(renamePendingFileInfo.destination,
+ renamePendingFileInfo.src);
+ isReDone = true;
+ } else {
+ isReDone = false;
+ }
+ }
+
+ private RenamePendingFileInfo readFile(final Path redoFile,
+ final AbfsInputStream redoFileInputStream)
+ throws IOException {
+ Path f = redoFile;
+ byte[] bytes = new byte[MAX_RENAME_PENDING_FILE_SIZE];
+ int l = redoFileInputStream.read(bytes);
+ if (l <= 0) {
+ // Jira HADOOP-12678 -Handle empty rename pending metadata file during
+ // atomic rename in redo path. If during renamepending file is created
+ // but not written yet, then this means that rename operation
+ // has not started yet. So we should delete rename pending metadata file.
+ LOG.error("Deleting empty rename pending file "
+ + redoFile + " -- no data available");
+ deleteRenamePendingFile(azureBlobFileSystem, redoFile);
+ return null;
+ }
+ if (l == MAX_RENAME_PENDING_FILE_SIZE) {
+ throw new IOException(
+ "Error reading pending rename file contents -- "
+ + "maximum file size exceeded");
+ }
+ String contents = new String(bytes, 0, l, StandardCharsets.UTF_8);
+
+ // parse the JSON
+ JsonNode json = null;
+ boolean committed;
+ try {
+ json = READER.readValue(contents);
+ committed = true;
+ } catch (JsonMappingException e) {
+
+ // The -RedoPending.json file is corrupted, so we assume it was
+ // not completely written
+ // and the redo operation did not commit.
+ committed = false;
+ } catch (JsonParseException e) {
+ committed = false;
+ }
+
+ if (!committed) {
+ LOG.error("Deleting corruped rename pending file {} \n {}",
+ redoFile, contents);
+
+ // delete the -RenamePending.json file
+ deleteRenamePendingFile(azureBlobFileSystem, redoFile);
+ return null;
+ }
+
+ // initialize this object's fields
+ JsonNode oldFolderName = json.get("OldFolderName");
+ JsonNode newFolderName = json.get("NewFolderName");
+ JsonNode eTag = json.get("ETag");
+
+ if (oldFolderName != null && StringUtils.isNotEmpty(
+ oldFolderName.textValue())
+ && newFolderName != null && StringUtils.isNotEmpty(
+ newFolderName.textValue()) && eTag != null && StringUtils.isNotEmpty(
+ eTag.textValue())) {
+ RenamePendingFileInfo renamePendingFileInfo = new RenamePendingFileInfo();
+ renamePendingFileInfo.destination = new Path(newFolderName.textValue());
+ renamePendingFileInfo.src = new Path(oldFolderName.textValue());
+ renamePendingFileInfo.eTag = eTag.textValue();
+ return renamePendingFileInfo;
+ }
+ return null;
+ }
+
+ private void deleteRenamePendingFile(FileSystem fs, Path redoFile)
+ throws IOException {
+ try {
+ fs.delete(redoFile, false);
+ } catch (IOException e) {
+ // If the rename metadata was not found then somebody probably
+ // raced with us and finished the delete first
+ if (e instanceof FileNotFoundException) {
+ LOG.warn("rename pending file " + redoFile + " is already deleted");
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ /**
+ * Write to disk the information needed to redo folder rename,
+ * in JSON format. The file name will be
+ * {@code abfs:///folderName-RenamePending.json}
+ * The file format will be:
+ * {@code
+ * {
+ * FormatVersion: "1.0",
+ * OperationTime: "",
+ * OldFolderName: "",
+ * NewFolderName: ""
+ * ETag: ""
+ * }
+ *
+ * Here's a sample:
+ * {
+ * FormatVersion: "1.0",
+ * OperationUTCTime: "2014-07-01 23:50:35.572",
+ * OldFolderName: "user/ehans/folderToRename",
+ * NewFolderName: "user/ehans/renamedFolder"
+ * ETag: "ETag"
+ * } }
+ * @throws IOException Thrown when fail to write file.
+ */
+ public void preRename(final Boolean isCreateOperationOnBlobEndpoint,
+ final String eTag) throws IOException {
+ Path path = getRenamePendingFilePath();
+ LOG.debug("Preparing to write atomic rename state to {}", path.toString());
+ OutputStream output = null;
+
+ String contents = makeRenamePendingFileContents(eTag);
+
+ // Write file.
+ try {
+ output = azureBlobFileSystem.create(path, false);
+ output.write(contents.getBytes(Charset.forName("UTF-8")));
+ output.flush();
+ output.close();
+ } catch (IOException e) {
+ /*
+ * Scenario: file has been deleted by parallel thread before the RenameJSON
+ * could be written and flushed.
+ * ref: https://issues.apache.org/jira/browse/HADOOP-12678
+ * On DFS endpoint, flush API is called. If file is not there, server returns
+ * 404.
+ * On blob endpoint, flush API is not there. PutBlockList is called with
+ * if-match header. If file is not there, the conditional header will fail,
+ * the server will return 412.
+ */
+ if ((!isCreateOperationOnBlobEndpoint
+ && e instanceof FileNotFoundException) || (
+ isCreateOperationOnBlobEndpoint && getWrappedException(
+ e) instanceof AbfsRestOperationException &&
+ ((AbfsRestOperationException) getWrappedException(
+ e)).getStatusCode()
+ == HttpURLConnection.HTTP_PRECON_FAILED)) {
+ /*
+ * In case listStatus done on directory before any content could be written,
+ * that particular thread running on some worker-node of the cluster would
+ * delete the RenamePending JSON file.
+ * ref: https://issues.apache.org/jira/browse/HADOOP-12678.
+ * To recover from parallel delete, we will give it a second try.
+ */
+ output = azureBlobFileSystem.create(path, false);
+ output.write(contents.getBytes(Charset.forName("UTF-8")));
+ output.flush();
+ output.close();
+ return;
+ }
+ throw new IOException(
+ "Unable to write RenamePending file for folder rename from "
+ + srcPath.toUri().getPath() + " to " + dstPath.toUri().getPath(),
+ e);
+ }
+ }
+
+ private Throwable getWrappedException(final IOException e) {
+ if (e.getCause() != null) {
+ return e.getCause().getCause();
+ }
+ return e;
+ }
+
+ /**
+ * Return the contents of the JSON file to represent the operations
+ * to be performed for a folder rename.
+ *
+ * @return JSON string which represents the operation.
+ */
+ private String makeRenamePendingFileContents(String eTag) {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
+ sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
+ String time = sdf.format(new Date());
+ if(!eTag.startsWith("\"") && !eTag.endsWith("\"")) {
+ eTag = quote(eTag);
+ }
+
+ // Make file contents as a string. Again, quote file names, escaping
+ // characters as appropriate.
+ String contents = "{\n"
+ + " FormatVersion: \"1.0\",\n"
+ + " OperationUTCTime: \"" + time + "\",\n"
+ + " OldFolderName: " + quote(srcPath.toUri().getPath()) + ",\n"
+ + " NewFolderName: " + quote(dstPath.toUri().getPath()) + ",\n"
+ + " ETag: " + eTag + "\n"
+ + "}\n";
+
+ return contents;
+ }
+
+ /**
+ * This is an exact copy of org.codehaus.jettison.json.JSONObject.quote
+ * method.
+ *
+ * Produce a string in double quotes with backslash sequences in all the
+ * right places. A backslash will be inserted within
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.ClientErrorRetryReason;
+import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.ConnectionResetRetryReason;
+import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.ConnectionTimeoutRetryReason;
+import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.ReadTimeoutRetryReason;
+import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.RetryReasonCategory;
+import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.ServerErrorRetryReason;
+import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.UnknownHostRetryReason;
+import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.UnknownIOExceptionRetryReason;
+import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.UnknownSocketExceptionRetryReason;
+
+
+/**
+ * This utility class exposes methods to convert a server response-error to a
+ * category of error.
+ */
+final class RetryReason {
+
+ /**
+ * Linked-list of the implementations of RetryReasonCategory. The objects in the
+ * list are arranged by the rank of their significance.
+ *
+ * - ServerError (statusCode==5XX), ClientError (statusCode==4XX) are
+ * independent of other retryReason categories.
+ * - Since {@link java.net.SocketException} is subclass of
+ * {@link java.io.IOException},
+ * hence, {@link UnknownIOExceptionRetryReason} is placed before
+ * {@link UnknownSocketExceptionRetryReason}
+ * - Since, connectionTimeout, readTimeout, and connectionReset are
+ * {@link java.net.SocketTimeoutException} exceptions with different messages,
+ * hence, {@link ConnectionTimeoutRetryReason}, {@link ReadTimeoutRetryReason},
+ * {@link ConnectionResetRetryReason} are above {@link UnknownIOExceptionRetryReason}.
+ * There is no order between the three reasons as they are differentiated
+ * by exception-message.
+ * - Since, {@link java.net.UnknownHostException} is subclass of
+ * {@link java.io.IOException}, {@link UnknownHostRetryReason} is placed
+ * over {@link UnknownIOExceptionRetryReason}
+ *
+ */
+ private static List rankedReasonCategories
+ = new LinkedList() {{
+ add(new ServerErrorRetryReason());
+ add(new ClientErrorRetryReason());
+ add(new UnknownIOExceptionRetryReason());
+ add(new UnknownSocketExceptionRetryReason());
+ add(new ConnectionTimeoutRetryReason());
+ add(new ReadTimeoutRetryReason());
+ add(new UnknownHostRetryReason());
+ add(new ConnectionResetRetryReason());
+ }};
+
+ private RetryReason() {
+
+ }
+
+ /**
+ * Method to get correct abbreviation for a given set of exception, statusCode,
+ * storageStatusCode.
+ *
+ * @param ex exception caught during server communication.
+ * @param statusCode statusCode in the server response.
+ * @param storageErrorMessage storageErrorMessage in the server response.
+ *
+ * @return abbreviation for the the given set of exception, statusCode, storageStatusCode.
+ */
+ static String getAbbreviation(Exception ex,
+ Integer statusCode,
+ String storageErrorMessage) {
+ String result = null;
+ for (RetryReasonCategory retryReasonCategory : rankedReasonCategories) {
+ final String abbreviation
+ = retryReasonCategory.captureAndGetAbbreviation(ex,
+ statusCode, storageErrorMessage);
+ if (abbreviation != null) {
+ result = abbreviation;
+ }
+ }
+ return result;
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/RetryReasonConstants.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/RetryReasonConstants.java
new file mode 100644
index 00000000000000..8a0af183e30aea
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/RetryReasonConstants.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+public final class RetryReasonConstants {
+
+ private RetryReasonConstants() {
+
+ }
+ public static final String CONNECTION_TIMEOUT_JDK_MESSAGE = "connect timed out";
+ public static final String READ_TIMEOUT_JDK_MESSAGE = "Read timed out";
+ public static final String CONNECTION_RESET_MESSAGE = "Connection reset";
+ public static final String OPERATION_BREACH_MESSAGE = "Operations per second is over the account limit.";
+ public static final String CONNECTION_RESET_ABBREVIATION = "CR";
+ public static final String CONNECTION_TIMEOUT_ABBREVIATION = "CT";
+ public static final String READ_TIMEOUT_ABBREVIATION = "RT";
+ public static final String INGRESS_LIMIT_BREACH_ABBREVIATION = "ING";
+ public static final String EGRESS_LIMIT_BREACH_ABBREVIATION = "EGR";
+ public static final String OPERATION_LIMIT_BREACH_ABBREVIATION = "OPR";
+ public static final String UNKNOWN_HOST_EXCEPTION_ABBREVIATION = "UH";
+ public static final String IO_EXCEPTION_ABBREVIATION = "IOE";
+ public static final String SOCKET_EXCEPTION_ABBREVIATION = "SE";
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/TimerFunctionality.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/TimerFunctionality.java
new file mode 100644
index 00000000000000..52428fdd54a19f
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/TimerFunctionality.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+public enum TimerFunctionality {
+ RESUME,
+
+ SUSPEND
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ClientErrorRetryReason.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ClientErrorRetryReason.java
new file mode 100644
index 00000000000000..cf1c47e3eb0dc5
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ClientErrorRetryReason.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
+
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_STATUS_CATEGORY_QUOTIENT;
+
+/**
+ * Category that can capture server-response errors for 4XX status-code.
+ */
+public class ClientErrorRetryReason extends RetryReasonCategory {
+
+ @Override
+ Boolean canCapture(final Exception ex,
+ final Integer statusCode,
+ final String serverErrorMessage) {
+ if (statusCode == null || statusCode / HTTP_STATUS_CATEGORY_QUOTIENT != 4) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ String getAbbreviation(final Integer statusCode,
+ final String serverErrorMessage) {
+ return statusCode + "";
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ConnectionResetRetryReason.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ConnectionResetRetryReason.java
new file mode 100644
index 00000000000000..702f8875646324
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ConnectionResetRetryReason.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
+
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_RESET_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_RESET_MESSAGE;
+
+/**
+ * Category that can capture server-response errors for connection-reset exception.
+ */
+public class ConnectionResetRetryReason extends
+ RetryReasonCategory {
+
+ @Override
+ Boolean canCapture(final Exception ex,
+ final Integer statusCode,
+ final String serverErrorMessage) {
+ return checkExceptionMessage(ex, CONNECTION_RESET_MESSAGE);
+ }
+
+ @Override
+ String getAbbreviation(final Integer statusCode,
+ final String serverErrorMessage) {
+ return CONNECTION_RESET_ABBREVIATION;
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ConnectionTimeoutRetryReason.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ConnectionTimeoutRetryReason.java
new file mode 100644
index 00000000000000..28f35dcc805468
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ConnectionTimeoutRetryReason.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
+
+
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_TIMEOUT_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_TIMEOUT_JDK_MESSAGE;
+
+/**
+ * Category that can capture server-response errors for connection-timeout.
+ */
+public class ConnectionTimeoutRetryReason extends
+ RetryReasonCategory {
+
+ @Override
+ String getAbbreviation(final Integer statusCode,
+ final String serverErrorMessage) {
+ return CONNECTION_TIMEOUT_ABBREVIATION;
+ }
+
+ @Override
+ Boolean canCapture(final Exception ex,
+ final Integer statusCode,
+ final String serverErrorMessage) {
+ return checkExceptionMessage(ex, CONNECTION_TIMEOUT_JDK_MESSAGE);
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ReadTimeoutRetryReason.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ReadTimeoutRetryReason.java
new file mode 100644
index 00000000000000..4663d9a52bbd6c
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ReadTimeoutRetryReason.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
+
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.READ_TIMEOUT_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.READ_TIMEOUT_JDK_MESSAGE;
+
+/**
+ * Category that can capture server-response errors for read-timeout.
+ */
+public class ReadTimeoutRetryReason extends RetryReasonCategory {
+
+ @Override
+ Boolean canCapture(final Exception ex,
+ final Integer statusCode,
+ final String serverErrorMessage) {
+ return checkExceptionMessage(ex, READ_TIMEOUT_JDK_MESSAGE);
+ }
+
+ @Override
+ String getAbbreviation(final Integer statusCode,
+ final String serverErrorMessage) {
+ return READ_TIMEOUT_ABBREVIATION;
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/RetryReasonCategory.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/RetryReasonCategory.java
new file mode 100644
index 00000000000000..893451b496f457
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/RetryReasonCategory.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
+
+import java.util.Locale;
+
+/**
+ * Provides methods to define if given exception can be categorised to certain category.
+ * Each category has a different implementation of the abstract class.
+ */
+public abstract class RetryReasonCategory {
+
+ /**
+ * Returns if given server response error can be categorised by the implementation.
+ *
+ * @param ex exception captured in the server response.
+ * @param statusCode statusCode on the server response
+ * @param serverErrorMessage serverErrorMessage on the server response.
+ *
+ * @return
- true if server response error can be categorised by the implementation
+ * - false if response error can not be categorised by the implementation
+ */
+ abstract Boolean canCapture(Exception ex,
+ Integer statusCode,
+ String serverErrorMessage);
+
+ /**
+ * Returns the abbreviation corresponding to the server response error.
+ *
+ * @param statusCode statusCode on the server response
+ * @param serverErrorMessage serverErrorMessage on the server response.
+ *
+ * @return abbreviation on the basis of the statusCode and the serverErrorMessage
+ */
+ abstract String getAbbreviation(Integer statusCode, String serverErrorMessage);
+
+ /**
+ * Converts the server-error response to an abbreviation if the response can be
+ * categorised by the implementation.
+ *
+ * @param ex exception received while making API request
+ * @param statusCode statusCode received in the server-response
+ * @param serverErrorMessage error-message received in the server-response
+ *
+ * @return abbreviation if the server-response can be categorised by the implementation.
+ * null if the server-response can not be categorised by the implementation.
+ */
+ public String captureAndGetAbbreviation(Exception ex,
+ Integer statusCode,
+ String serverErrorMessage) {
+ if (canCapture(ex, statusCode, serverErrorMessage)) {
+ return getAbbreviation(statusCode, serverErrorMessage);
+ }
+ return null;
+ }
+
+ /**
+ * Checks if a required search-string is in the exception's message.
+ */
+ Boolean checkExceptionMessage(final Exception exceptionCaptured,
+ final String search) {
+ if (search == null) {
+ return false;
+ }
+ if (exceptionCaptured != null
+ && exceptionCaptured.getMessage() != null
+ && exceptionCaptured.getMessage()
+ .toLowerCase(Locale.US)
+ .contains(search.toLowerCase(Locale.US))) {
+ return true;
+ }
+ return false;
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ServerErrorRetryReason.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ServerErrorRetryReason.java
new file mode 100644
index 00000000000000..dd67a0cb8cbba2
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ServerErrorRetryReason.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
+
+import static java.net.HttpURLConnection.HTTP_UNAVAILABLE;
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_STATUS_CATEGORY_QUOTIENT;
+import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.EGRESS_OVER_ACCOUNT_LIMIT;
+import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.EGRESS_LIMIT_BREACH_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.INGRESS_LIMIT_BREACH_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.OPERATION_BREACH_MESSAGE;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.OPERATION_LIMIT_BREACH_ABBREVIATION;
+
+/**
+ * Category that can capture server-response errors for 5XX status-code.
+ */
+public class ServerErrorRetryReason extends RetryReasonCategory {
+
+ @Override
+ Boolean canCapture(final Exception ex,
+ final Integer statusCode,
+ final String serverErrorMessage) {
+ if (statusCode == null || statusCode / HTTP_STATUS_CATEGORY_QUOTIENT != 5) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ String getAbbreviation(final Integer statusCode,
+ final String serverErrorMessage) {
+ if (statusCode == HTTP_UNAVAILABLE && serverErrorMessage != null) {
+ String splitedServerErrorMessage = serverErrorMessage.split(System.lineSeparator(),
+ 2)[0];
+ if (INGRESS_OVER_ACCOUNT_LIMIT.getErrorMessage().equalsIgnoreCase(
+ splitedServerErrorMessage)) {
+ return INGRESS_LIMIT_BREACH_ABBREVIATION;
+ }
+ if (EGRESS_OVER_ACCOUNT_LIMIT.getErrorMessage().equalsIgnoreCase(
+ splitedServerErrorMessage)) {
+ return EGRESS_LIMIT_BREACH_ABBREVIATION;
+ }
+ if (OPERATION_BREACH_MESSAGE.equalsIgnoreCase(
+ splitedServerErrorMessage)) {
+ return OPERATION_LIMIT_BREACH_ABBREVIATION;
+ }
+ return HTTP_UNAVAILABLE + "";
+ }
+ return statusCode + "";
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/UnknownHostRetryReason.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/UnknownHostRetryReason.java
new file mode 100644
index 00000000000000..c329348d81f8d6
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/UnknownHostRetryReason.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
+
+import java.net.UnknownHostException;
+
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.UNKNOWN_HOST_EXCEPTION_ABBREVIATION;
+
+/**
+ * Category that can capture server-response errors for {@link UnknownHostException}.
+ */
+public class UnknownHostRetryReason extends RetryReasonCategory {
+
+ @Override
+ Boolean canCapture(final Exception ex,
+ final Integer statusCode,
+ final String serverErrorMessage) {
+ if (ex instanceof UnknownHostException) {
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ String getAbbreviation(final Integer statusCode,
+ final String serverErrorMessage) {
+ return UNKNOWN_HOST_EXCEPTION_ABBREVIATION;
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/UnknownIOExceptionRetryReason.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/UnknownIOExceptionRetryReason.java
new file mode 100644
index 00000000000000..8a69ebb928d684
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/UnknownIOExceptionRetryReason.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.IO_EXCEPTION_ABBREVIATION;
+
+
+/**
+ * Category that can capture server-response errors for {@link IOException}.
+ */
+public class UnknownIOExceptionRetryReason extends
+ RetryReasonCategory {
+
+ @Override
+ Boolean canCapture(final Exception ex,
+ final Integer statusCode,
+ final String serverErrorMessage) {
+ if (ex instanceof IOException) {
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ String getAbbreviation(final Integer statusCode,
+ final String serverErrorMessage) {
+ return IO_EXCEPTION_ABBREVIATION;
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/UnknownSocketExceptionRetryReason.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/UnknownSocketExceptionRetryReason.java
new file mode 100644
index 00000000000000..18e9f115feaf69
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/UnknownSocketExceptionRetryReason.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
+
+import java.net.SocketException;
+
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.SOCKET_EXCEPTION_ABBREVIATION;
+
+/**
+ * Category that can capture server-response errors for {@link SocketException}.
+ */
+public class UnknownSocketExceptionRetryReason extends
+ RetryReasonCategory {
+
+ @Override
+ Boolean canCapture(final Exception ex,
+ final Integer statusCode,
+ final String serverErrorMessage) {
+ if (ex instanceof SocketException) {
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ String getAbbreviation(final Integer statusCode,
+ final String serverErrorMessage) {
+ return SOCKET_EXCEPTION_ABBREVIATION;
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/package-info.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/package-info.java
new file mode 100644
index 00000000000000..7d8078620af777
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/package-info.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A retryReasonCategory defines methods applicable on server-response errors.
+ */
+@Private
+@Evolving
+package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java
index 5a115451df159b..e2ab95aec91028 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java
@@ -61,12 +61,25 @@ public class TracingContext {
private final TracingHeaderFormat format; // header ID display options
private Listener listener = null; // null except when testing
//final concatenated ID list set into x-ms-client-request-id header
+ private String fallbackDFSAppend = "B";
private String header = EMPTY_STRING;
+ /**
+ * If {@link #primaryRequestId} is null, this field shall be set equal
+ * to the last part of the {@link #clientRequestId}'s UUID
+ * in {@link #constructHeader(AbfsHttpOperation, String)} only on the
+ * first API call for an operation. Subsequent retries for that operation
+ * will not change this field. In case {@link #primaryRequestId} is non-null,
+ * this field shall not be set.
+ */
+ private String primaryRequestIdForRetry;
+
private static final Logger LOG = LoggerFactory.getLogger(AbfsClient.class);
public static final int MAX_CLIENT_CORRELATION_ID_LENGTH = 72;
public static final String CLIENT_CORRELATION_ID_PATTERN = "[a-zA-Z0-9-]*";
+ private Integer operatedBlobCount = null;
+
/**
* Initialize TracingContext
* @param clientCorrelationID Provided over config by client
@@ -139,6 +152,10 @@ public void setOperation(FSOperationType operation) {
this.opType = operation;
}
+ public int getRetryCount() {
+ return retryCount;
+ }
+
public void setRetryCount(int retryCount) {
this.retryCount = retryCount;
}
@@ -147,20 +164,34 @@ public void setListener(Listener listener) {
this.listener = listener;
}
+ public void setFallbackDFSAppend(String fallbackDFSAppend) {
+ this.fallbackDFSAppend = fallbackDFSAppend;
+ }
+
+ public String getFallbackDFSAppend() {
+ return fallbackDFSAppend;
+ }
+
/**
* Concatenate all identifiers separated by (:) into a string and set into
* X_MS_CLIENT_REQUEST_ID header of the http operation
* @param httpOperation AbfsHttpOperation instance to set header into
* connection
+ * @param previousFailure Failure seen before this API trigger on same operation
+ * from AbfsClient.
*/
- public void constructHeader(AbfsHttpOperation httpOperation) {
+ public void constructHeader(AbfsHttpOperation httpOperation, String previousFailure) {
clientRequestId = UUID.randomUUID().toString();
switch (format) {
case ALL_ID_FORMAT: // Optional IDs (e.g. streamId) may be empty
header =
clientCorrelationID + ":" + clientRequestId + ":" + fileSystemID + ":"
- + primaryRequestId + ":" + streamID + ":" + opType + ":"
- + retryCount;
+ + getPrimaryRequestIdForHeader(retryCount > 0) + ":" + streamID
+ + ":" + opType + ":" + retryCount;
+ header = addFailureReasons(header, previousFailure) + ":" + fallbackDFSAppend;
+ if (operatedBlobCount != null) {
+ header += (":" + operatedBlobCount);
+ }
break;
case TWO_ID_FORMAT:
header = clientCorrelationID + ":" + clientRequestId;
@@ -172,6 +203,39 @@ public void constructHeader(AbfsHttpOperation httpOperation) {
listener.callTracingHeaderValidator(header, format);
}
httpOperation.setRequestProperty(HttpHeaderConfigurations.X_MS_CLIENT_REQUEST_ID, header);
+ /*
+ * In case the primaryRequestId is an empty-string and if it is the first try to
+ * API call (previousFailure shall be null), maintain the last part of clientRequestId's
+ * UUID in primaryRequestIdForRetry. This field shall be used as primaryRequestId part
+ * of the x-ms-client-request-id header in case of retry of the same API-request.
+ */
+ if (primaryRequestId.isEmpty() && previousFailure == null) {
+ String[] clientRequestIdParts = clientRequestId.split("-");
+ primaryRequestIdForRetry = clientRequestIdParts[
+ clientRequestIdParts.length - 1];
+ }
+ }
+
+ /**
+ * Provide value to be used as primaryRequestId part of x-ms-client-request-id header.
+ * @param isRetry define if it's for a retry case.
+ * @return {@link #primaryRequestIdForRetry}:If the {@link #primaryRequestId}
+ * is an empty-string, and it's a retry iteration.
+ * {@link #primaryRequestId} for other cases.
+ */
+ private String getPrimaryRequestIdForHeader(final Boolean isRetry) {
+ if (!primaryRequestId.isEmpty() || !isRetry) {
+ return primaryRequestId;
+ }
+ return primaryRequestIdForRetry;
+ }
+
+ private String addFailureReasons(final String header,
+ final String previousFailure) {
+ if (previousFailure == null) {
+ return header;
+ }
+ return String.format("%s_%s", header, previousFailure);
}
/**
@@ -182,4 +246,19 @@ public String getHeader() {
return header;
}
+ public String getPrimaryRequestId() {
+ return primaryRequestId;
+ }
+
+ public void setOperatedBlobCount(Integer count) {
+ operatedBlobCount = count;
+ }
+
+ public Integer getOperatedBlobCount() {
+ return operatedBlobCount;
+ }
+
+ public FSOperationType getOpType() {
+ return opType;
+ }
}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java
index e27d54b443ca20..857b20b2fd80bd 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java
@@ -169,6 +169,20 @@ public static String getMaskedUrl(URL url) {
return url.toString().replace(queryString, maskedQueryString);
}
+ public static String encodeMetadataAttribute(String value) throws UnsupportedEncodingException {
+ // We have to URL encode the attribute as it could
+ // have URI special characters which unless encoded will result
+ // in 403 errors from the server. This is due to metadata properties
+ // being sent in the HTTP header of the request which is in turn used
+ // on the server side to authorize the request.
+ return value == null ? null : URLEncoder.encode(value, StandardCharsets.UTF_8.name());
+ }
+
+ public static String decodeMetadataAttribute(String encoded) throws UnsupportedEncodingException {
+ return encoded == null ? null :
+ java.net.URLDecoder.decode(encoded, StandardCharsets.UTF_8.name());
+ }
+
private UriUtils() {
}
}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/WeakReferenceMap.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/WeakReferenceMap.java
new file mode 100644
index 00000000000000..cd47809689c416
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/WeakReferenceMap.java
@@ -0,0 +1,333 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.utils;
+
+import java.lang.ref.WeakReference;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Consumer;
+import java.util.function.Function;
+
+import javax.annotation.Nullable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.store.LogExactlyOnce;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * A map of keys type K to objects of type V which uses weak references,
+ * so does lot leak memory through long-lived references
+ * at the expense of losing references when GC takes place..
+ *
+ * This class is intended be used instead of ThreadLocal storage when
+ * references are to be cleaned up when the instance holding.
+ * In this use case, the key is the Long key.
+ *
+ * Concurrency.
+ * The class assumes that map entries are rarely contended for when writing,
+ * and that not blocking other threads is more important than atomicity.
+ * - a ConcurrentHashMap is used to map keys to weak references, with
+ * all its guarantees.
+ * - there is no automatic pruning.
+ * - see {@link #create(Object)} for the concurrency semantics on entry creation.
+ */
+@InterfaceAudience.Private
+public class WeakReferenceMap {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(WeakReferenceMap.class);
+
+ /**
+ * The reference map.
+ */
+ private final Map> map = new ConcurrentHashMap<>();
+
+ /**
+ * Supplier of new instances.
+ */
+ private final Function factory;
+
+ /**
+ * Nullable callback when a get on a key got a weak reference back.
+ * The assumption is that this is for logging/stats, which is why
+ * no attempt is made to use the call as a supplier of a new value.
+ */
+ private final Consumer referenceLost;
+
+ /**
+ * Counter of references lost.
+ */
+ private final AtomicLong referenceLostCount = new AtomicLong();
+
+ /**
+ * Counter of entries created.
+ */
+ private final AtomicLong entriesCreatedCount = new AtomicLong();
+
+ /**
+ * Log to report loss of a reference during the create phase, which
+ * is believed to be a cause of HADOOP-18456.
+ */
+ private final LogExactlyOnce referenceLostDuringCreation = new LogExactlyOnce(LOG);
+
+ /**
+ * instantiate.
+ * @param factory supplier of new instances
+ * @param referenceLost optional callback on lost references.
+ */
+ public WeakReferenceMap(
+ Function factory,
+ @Nullable final Consumer referenceLost) {
+
+ this.factory = requireNonNull(factory);
+ this.referenceLost = referenceLost;
+ }
+
+ @Override
+ public String toString() {
+ return "WeakReferenceMap{" +
+ "size=" + size() +
+ ", referenceLostCount=" + referenceLostCount +
+ ", entriesCreatedCount=" + entriesCreatedCount +
+ '}';
+ }
+
+ /**
+ * Map size.
+ * @return the current map size.
+ */
+ public int size() {
+ return map.size();
+ }
+
+ /**
+ * Clear all entries.
+ */
+ public void clear() {
+ map.clear();
+ }
+
+ /**
+ * look up the value, returning the possibly empty weak reference
+ * to a value, or null if no value was found.
+ * @param key key to look up
+ * @return null if there is no entry, a weak reference if found
+ */
+ public WeakReference lookup(K key) {
+ return map.get(key);
+ }
+
+ /**
+ * Get the value, creating if needed.
+ * @param key key.
+ * @return an instance.
+ */
+ public V get(K key) {
+ final WeakReference currentWeakRef = lookup(key);
+ // resolve it, after which if not null, we have a strong reference
+ V strongVal = resolve(currentWeakRef);
+ if (strongVal != null) {
+ // all good.
+ return strongVal;
+ }
+
+ // here, either currentWeakRef was null, or its reference was GC'd.
+ if (currentWeakRef != null) {
+ // garbage collection removed the reference.
+
+ // explicitly remove the weak ref from the map if it has not
+ // been updated by this point
+ // this is here just for completeness.
+ map.remove(key, currentWeakRef);
+
+ // log/report the loss.
+ noteLost(key);
+ }
+
+ // create a new value and add it to the map
+ return create(key);
+ }
+
+ /**
+ * Create a new instance under a key.
+ *
+ * The instance is created, added to the map and then the
+ * map value retrieved.
+ * This ensures that the reference returned is that in the map,
+ * even if there is more than one entry being created at the same time.
+ * If that race does occur, it will be logged the first time it happens
+ * for this specific map instance.
+ *
+ * HADOOP-18456 highlighted the risk of a concurrent GC resulting a null
+ * value being retrieved and so returned.
+ * To prevent this:
+ *
+ * - A strong reference is retained to the newly created instance
+ * in a local variable.
+ * - That variable is used after the resolution process, to ensure
+ * the JVM doesn't consider it "unreachable" and so eligible for GC.
+ * - A check is made for the resolved reference being null, and if so,
+ * the put() is repeated
+ *
+ * @param key key
+ * @return the created value
+ */
+ public V create(K key) {
+ entriesCreatedCount.incrementAndGet();
+ /*
+ Get a strong ref so even if a GC happens in this method the reference is not lost.
+ It is NOT enough to have a reference in a field, it MUST be used
+ so as to ensure the reference isn't optimized away prematurely.
+ "A reachable object is any object that can be accessed in any potential continuing
+ computation from any live thread."
+ */
+
+ final V strongRef = requireNonNull(factory.apply(key),
+ "factory returned a null instance");
+ V resolvedStrongRef;
+ do {
+ WeakReference newWeakRef = new WeakReference<>(strongRef);
+
+ // put it in the map
+ map.put(key, newWeakRef);
+
+ // get it back from the map
+ WeakReference retrievedWeakRef = map.get(key);
+ // resolve that reference, handling the situation where somehow it was removed from the map
+ // between the put() and the get()
+ resolvedStrongRef = resolve(retrievedWeakRef);
+ if (resolvedStrongRef == null) {
+ referenceLostDuringCreation.warn("reference to %s lost during creation", key);
+ noteLost(key);
+ }
+ } while (resolvedStrongRef == null);
+
+ // note if there was any change in the reference.
+ // as this forces strongRef to be kept in scope
+ if (strongRef != resolvedStrongRef) {
+ LOG.debug("Created instance for key {}: {} overwritten by {}",
+ key, strongRef, resolvedStrongRef);
+ }
+
+ return resolvedStrongRef;
+ }
+
+ /**
+ * Put a value under the key.
+ * A null value can be put, though on a get() call
+ * a new entry is generated
+ *
+ * @param key key
+ * @param value value
+ * @return any old non-null reference.
+ */
+ public V put(K key, V value) {
+ return resolve(map.put(key, new WeakReference<>(value)));
+ }
+
+ /**
+ * Remove any value under the key.
+ * @param key key
+ * @return any old non-null reference.
+ */
+ public V remove(K key) {
+ return resolve(map.remove(key));
+ }
+
+ /**
+ * Does the map have a valid reference for this object?
+ * no-side effects: there's no attempt to notify or cleanup
+ * if the reference is null.
+ * @param key key to look up
+ * @return true if there is a valid reference.
+ */
+ public boolean containsKey(K key) {
+ final WeakReference current = lookup(key);
+ return resolve(current) != null;
+ }
+
+ /**
+ * Given a possibly null weak reference, resolve
+ * its value.
+ * @param r reference to resolve
+ * @return the value or null
+ */
+ protected V resolve(WeakReference r) {
+ return r == null ? null : r.get();
+ }
+
+ /**
+ * Prune all null weak references, calling the referenceLost
+ * callback for each one.
+ *
+ * non-atomic and non-blocking.
+ * @return the number of entries pruned.
+ */
+ public int prune() {
+ int count = 0;
+ final Iterator>> it = map.entrySet().iterator();
+ while (it.hasNext()) {
+ final Map.Entry> next = it.next();
+ if (next.getValue().get() == null) {
+ it.remove();
+ count++;
+ noteLost(next.getKey());
+ }
+ }
+ return count;
+ }
+
+ /**
+ * Notify the reference lost callback.
+ * @param key key of lost reference
+ */
+ private void noteLost(final K key) {
+ // increment local counter
+ referenceLostCount.incrementAndGet();
+
+ // and call any notification function supplied in the constructor
+ if (referenceLost != null) {
+ referenceLost.accept(key);
+ }
+ }
+
+ /**
+ * Get count of references lost as detected
+ * during prune() or get() calls.
+ * @return count of references lost
+ */
+ public final long getReferenceLostCount() {
+ return referenceLostCount.get();
+ }
+
+ /**
+ * Get count of entries created on demand.
+ * @return count of entries created
+ */
+ public final long getEntriesCreatedCount() {
+ return entriesCreatedCount.get();
+ }
+}
+
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md
index dfb7f3f42a5cf9..ac77765f9e0b52 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md
@@ -769,6 +769,26 @@ Hflush() being the only documented API that can provide persistent data
transfer, Flush() also attempting to persist buffered data will lead to
performance issues.
+<<<<<<< HEAD
+=======
+### Hundred Continue Options
+
+`fs.azure.account.expect.header.enabled`: This configuration parameter is used
+to specify whether you wish to send a expect 100 continue header with each
+append request or not. It is configured to true by default. This flag configures
+the client to check with the Azure store before uploading a block of data from
+an output stream. This allows the client to throttle back gracefully -before
+actually attempting to upload the block. In experiments this provides
+significant throughput improvements under heavy load. For more information :
+- https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Expect
+
+
+### Account level throttling Options
+
+`fs.azure.account.operation.idle.timeout`: This value specifies the time after which the timer for the analyzer (read or
+write) should be paused until no new request is made again. The default value for the same is 60 seconds.
+
+>>>>>>> c88011c6046... HADOOP-18146: ABFS: Added changes for expect hundred continue header (#4039)
### HNS Check Options
Config `fs.azure.account.hns.enabled` provides an option to specify whether
the storage account is HNS enabled or not. In case the config is not provided,
@@ -874,6 +894,9 @@ when there are too many writes from the same process.
time. Effectively this will be the threadpool size within the
AbfsOutputStream instance. Set the value in between 1 to 8 both inclusive.
+`fs.azure.analysis.period`: The time after which sleep duration is recomputed after analyzing metrics. The default value
+for the same is 10 seconds.
+
`fs.azure.write.max.requests.to.queue`: To set the maximum write requests
that can be queued. Memory consumption of AbfsOutputStream instance can be
tuned with this config considering each queued request holds a buffer. Set
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md b/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
index 933f86be3e8961..e256fbef546cb2 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
@@ -602,26 +602,76 @@ various test combinations, it will:
2. Run tests for all combinations
3. Summarize results across all the test combination runs.
-As a pre-requisite step, fill config values for test accounts and credentials
-needed for authentication in `src/test/resources/azure-auth-keys.xml.template`
-and rename as `src/test/resources/azure-auth-keys.xml`.
+Below are the pre-requiste steps to follow:
+1. Copy
-**To add a new test combination:** Templates for mandatory test combinations
-for PR validation are present in `dev-support/testrun-scripts/runtests.sh`.
-If a new one needs to be added, add a combination set within
-`dev-support/testrun-scripts/runtests.sh` similar to the ones already defined
-and
-1. Provide a new combination name
-2. Update properties and values array which need to be effective for the test
-combination
-3. Call generateconfigs
+ ./src/test/resources/azure-auth-keys.xml.template
+ TO
+ ./src/test/resources/azure-auth-keys.xml
+ Update account names that should be used in the test run for HNS and non-HNS
+ combinations in the 2 properties present in the xml (account name should be
+ without domain part), namely
+
+ fs.azure.hnsTestAccountName
+ fs.azure.nonHnsTestAccountName
+ azure-auth-keys.xml is listed in .gitignore, so any accidental account name leak is prevented.
+
+```
+XInclude is supported, so for extra security secrets may be
+kept out of the source tree then referenced through an an XInclude element:
+
+
+```
+
+2. Create account config files (one config file per account) in folder:
+
+ ./src/test/resources/accountSettings/
+ Follow the instruction in the start of the template file
+
+ accountName_settings.xml.template
+ within accountSettings folder while creating account config file.
+ New files created in folder accountSettings is listed in .gitignore to
+ prevent accidental cred leaks.
**To run PR validation:** Running command
-* `dev-support/testrun-scripts/runtests.sh` will generate configurations for
-each of the combinations defined and run tests for all the combinations.
-* `dev-support/testrun-scripts/runtests.sh -c {combinationname}` Specific
-combinations can be provided with -c option. If combinations are provided
-with -c option, tests for only those combinations will be run.
+* `dev-support/testrun-scripts/runtests.sh` will prompt as below:
+```bash
+Choose action:
+[Note - SET_ACTIVE_TEST_CONFIG will help activate the config for IDE/single test class runs]
+1) SET_ACTIVE_TEST_CONFIG 4) SET_OR_CHANGE_TEST_ACCOUNT
+2) RUN_TEST 5) PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN
+3) CLEAN_UP_OLD_TEST_CONTAINERS
+#? 2
+```
+Enter 1: for setting active combination for IDE test run/single mvn test class runs.
+
+Enter 2: for choosing the combination to choose for mvn full test suite.
+
+Enter 3: For clean-up of any abruptly ending test leaving auto generated test
+container on the account.
+
+Enter 4: To create/modify the config file that decides the account to use for specific test combination.
+
+Enter 5: To print the log4j paths the last test runs.
+
+On next prompt, current list of combinations to choose are provided.
+Sample for Run_TEST action:
+```bash
+Enter parallel test run process count [default - 8]: 4
+Set the active test combination to run the action:
+1) HNS-OAuth 3) nonHNS-SharedKey 5) AllCombinationsTestRun
+2) HNS-SharedKey 4) AppendBlob-HNS-OAuth 6) Quit
+#? 1
+
+Combination specific property setting: [ key=fs.azure.account.auth.type , value=OAuth ]
+
+Activated [src/test/resources/abfs-combination-test-configs.xml] - for account: snvijayacontracttest for combination HNS-OAuth
+Running test for combination HNS-OAuth on account snvijayacontracttest [ProcessCount=4]
+Test run report can be seen in dev-support/testlogs/2022-10-07_05-23-22/Test-Logs-HNS-OAuth.txt
+````
+
+Provide the option for the action chosen first.
**Test logs:** Test runs will create a folder within dev-support/testlogs to
save the test logs. Folder name will be the test start timestamp. The mvn verify
@@ -632,25 +682,18 @@ consolidated results of all the combination runs will be saved into a file as
Test-Results.log in the same folder. When run for PR validation, the
consolidated test results needs to be pasted into the PR comment section.
-**To generate config for use in IDE:** Running command with -a (activate) option
-`dev-support/testrun-scripts/runtests.sh -a {combination name}` will update
-the effective config relevant for the specific test combination. Hence the same
-config files used by the mvn test runs can be used for IDE without any manual
-updates needed within config file.
-
-**Other command line options:**
-* -a Specify the combination name which needs to be
-activated. This is to be used to generate config for use in IDE.
-* -c Specify the combination name for test runs. If this
-config is specified, tests for only the specified combinations will run. All
-combinations of tests will be running if this config is not specified.
-* -t ABFS mvn tests are run in parallel mode. Tests by default
-are run with 8 thread count. It can be changed by providing -t
-
-In order to test ABFS, please add the following configuration to your
-`src/test/resources/azure-auth-keys.xml` file. Note that the ABFS tests include
-compatibility tests which require WASB credentials, in addition to the ABFS
-credentials.
+**To add a new test combination:** Templates for mandatory test combinations
+for PR validation are present in `dev-support/testrun-scripts/runtests.sh`.
+If a new one needs to be added, add a combination to
+`dev-support/testrun-scripts/runtests.sh`.
+(Refer to current active combinations within
+`SECTION: COMBINATION DEFINITIONS AND TRIGGER` and
+`SECTION: TEST COMBINATION METHODS` in the script).
+
+**Test Configuration Details:**
+
+ Note that the ABFS tests include compatibility tests which require WASB
+ credentials, in addition to the ABFS credentials.
```xml
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
index 5d2d5d4afdc3f4..72793eaecac04d 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
@@ -21,6 +21,7 @@
import com.microsoft.azure.storage.*;
import com.microsoft.azure.storage.blob.*;
import com.microsoft.azure.storage.core.Base64;
+import jdk.nashorn.internal.parser.Token;
import org.junit.Assert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -33,11 +34,14 @@
import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation;
import org.apache.hadoop.fs.azure.metrics.AzureFileSystemMetricsSystem;
import org.apache.hadoop.fs.azure.integration.AzureTestUtils;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException;
+import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider;
import org.apache.hadoop.metrics2.AbstractMetric;
import org.apache.hadoop.metrics2.MetricsRecord;
import org.apache.hadoop.metrics2.MetricsSink;
import org.apache.hadoop.metrics2.MetricsTag;
import org.apache.hadoop.metrics2.impl.TestMetricsConfig;
+import org.apache.kerby.kerberos.kerb.provider.TokenProvider;
import java.io.File;
import java.io.IOException;
@@ -71,6 +75,7 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable,
public static final String MOCK_CONTAINER_NAME = "mockContainer";
public static final String WASB_AUTHORITY_DELIMITER = "@";
public static final String WASB_SCHEME = "wasb";
+ public static final String WASBS_SCHEME = "wasbs";
public static final String PATH_DELIMITER = "/";
public static final String AZURE_ROOT_CONTAINER = "$root";
public static final String MOCK_WASB_URI = "wasb://" + MOCK_CONTAINER_NAME
@@ -456,13 +461,13 @@ public static void setMockAccountKey(Configuration conf, String accountName) {
private static URI createAccountUri(String accountName)
throws URISyntaxException {
- return new URI(WASB_SCHEME + ":" + PATH_DELIMITER + PATH_DELIMITER
+ return new URI(WASBS_SCHEME + ":" + PATH_DELIMITER + PATH_DELIMITER
+ accountName);
}
private static URI createAccountUri(String accountName, String containerName)
throws URISyntaxException {
- return new URI(WASB_SCHEME + ":" + PATH_DELIMITER + PATH_DELIMITER
+ return new URI(WASBS_SCHEME + ":" + PATH_DELIMITER + PATH_DELIMITER
+ containerName + WASB_AUTHORITY_DELIMITER + accountName);
}
@@ -505,6 +510,37 @@ static CloudStorageAccount createStorageAccount(String accountName,
credentials = new StorageCredentialsAccountAndKey(
accountName.split("\\.")[0], accountKey);
}
+ return new CloudStorageAccount(credentials);
+ }
+
+ static CloudStorageAccount createStorageAccount(String accountName,
+ Configuration conf, boolean allowAnonymous, NativeAzureFileSystem fs)
+ throws URISyntaxException,
+ KeyProviderException, IOException {
+ String accountKey = AzureNativeFileSystemStore
+ .getAccountKeyFromConfiguration(accountName, conf);
+ final StorageCredentials credentials;
+ if (fs.getStore().getTokenProviderStore() != null){
+ AccessTokenProvider tokenProvider = fs.getStore().getTokenProviderStore();
+ int iend = accountName.indexOf("."); //this finds the first occurrence of "."
+ if (iend != -1) {
+ accountName = accountName.substring(0 , iend); //this will give abc
+ }
+ credentials = new StorageCredentialsToken(accountName, tokenProvider.getToken().getAccessToken());
+ return new CloudStorageAccount(credentials, true);
+ }
+ if (accountKey == null) {
+ if (allowAnonymous) {
+ credentials = StorageCredentialsAnonymous.ANONYMOUS;
+ } else {
+ LOG.warn("Skipping live Azure test because of missing key for"
+ + " account '" + accountName + "'.");
+ return null;
+ }
+ } else {
+ credentials = new StorageCredentialsAccountAndKey(
+ accountName.split("\\.")[0], accountKey);
+ }
return new CloudStorageAccount(credentials);
}
@@ -540,6 +576,18 @@ static CloudStorageAccount createTestAccount(Configuration conf)
return createStorageAccount(testAccountName, conf, false);
}
+ static CloudStorageAccount createTestAccount(Configuration conf, NativeAzureFileSystem fs)
+ throws URISyntaxException, KeyProviderException, IOException {
+ AzureTestUtils.assumeNamespaceDisabled(conf);
+
+ String testAccountName = verifyWasbAccountNameInConfig(conf);
+ if (testAccountName == null) {
+ LOG.warn("Skipping live Azure test because of missing test account");
+ return null;
+ }
+ return createStorageAccount(testAccountName, conf, false, fs);
+ }
+
public static enum CreateOptions {
UseSas, Readonly, CreateContainer, useThrottling
}
@@ -565,15 +613,12 @@ public static AzureBlobStorageTestAccount create(
throws Exception {
saveMetricsConfigFile();
NativeAzureFileSystem fs = null;
+ fs = new NativeAzureFileSystem();
CloudBlobContainer container = null;
Configuration conf = createTestConfiguration(initialConfiguration);
configurePageBlobDir(conf);
configureAtomicRenameDir(conf);
- CloudStorageAccount account = createTestAccount(conf);
- if (account == null) {
- return null;
- }
- fs = new NativeAzureFileSystem();
+ CloudStorageAccount account = null;
String containerName = useContainerSuffixAsContainerName
? containerNameSuffix
: String.format(
@@ -581,44 +626,45 @@ public static AzureBlobStorageTestAccount create(
System.getProperty("user.name"),
UUID.randomUUID().toString(),
containerNameSuffix);
+ String accountName = verifyWasbAccountNameInConfig(conf);
+ // Check if throttling is turned on and set throttling parameters
+ // appropriately.
+ conf.setBoolean(KEY_DISABLE_THROTTLING,
+ !createOptions.contains(CreateOptions.useThrottling));
+
+// // Set account URI and initialize Azure file system.
+ URI accountUri = createAccountUri(accountName, containerName);
+ fs.initialize(accountUri, conf);
+ if (fs.getStore().getTokenProviderStore() != null){
+ account = createTestAccount(conf, fs);
+ } else {
+ account = createTestAccount(conf);
+ }
+ if (account == null) {
+ return null;
+ }
container = account.createCloudBlobClient().getContainerReference(
containerName);
if (createOptions.contains(CreateOptions.CreateContainer)) {
container.createIfNotExists();
}
- String accountName = verifyWasbAccountNameInConfig(conf);
if (createOptions.contains(CreateOptions.UseSas)) {
String sas = generateSAS(container,
createOptions.contains(CreateOptions.Readonly));
- if (!createOptions.contains(CreateOptions.CreateContainer)) {
- // The caller doesn't want the container to be pre-created,
- // so delete it now that we have generated the SAS.
- container.delete();
- }
- // Remove the account key from the configuration to make sure we don't
- // cheat and use that.
- // but only if not in secure mode, which requires that login
if (!conf.getBoolean(AzureNativeFileSystemStore.KEY_USE_SECURE_MODE, false)) {
conf.set(ACCOUNT_KEY_PROPERTY_NAME + accountName, "");
}
// Set the SAS key.
conf.set(SAS_PROPERTY_NAME + containerName + "." + accountName, sas);
+ fs.setConf(conf);
+ if (!createOptions.contains(CreateOptions.CreateContainer)) {
+ // The caller doesn't want the container to be pre-created,
+ // so delete it now that we have generated the SAS.
+ container.delete();
+ }
}
-
- // Check if throttling is turned on and set throttling parameters
- // appropriately.
- if (createOptions.contains(CreateOptions.useThrottling)) {
- conf.setBoolean(KEY_DISABLE_THROTTLING, false);
- } else {
- conf.setBoolean(KEY_DISABLE_THROTTLING, true);
- }
-
configureSecureModeTestSettings(conf);
- // Set account URI and initialize Azure file system.
- URI accountUri = createAccountUri(accountName, containerName);
- fs.initialize(accountUri, conf);
-
// Create test account initializing the appropriate member variables.
//
AzureBlobStorageTestAccount testAcct =
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemLive.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemLive.java
index f86af954933626..ca3deefff695ac 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemLive.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemLive.java
@@ -189,7 +189,7 @@ public void run() {
LOG.info("Freeing lease");
lease.free();
}
- } catch (StorageException se) {
+ } catch (StorageException | IOException se) {
LOG.warn("Unable to free lease.", se);
}
}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
index 6d11207c479b51..4f26777f512489 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
@@ -38,6 +38,7 @@
import org.apache.commons.codec.net.URLCodec;
import org.apache.commons.lang3.NotImplementedException;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider;
import org.apache.http.client.utils.URIBuilder;
import com.microsoft.azure.storage.AccessCondition;
@@ -178,6 +179,13 @@ public CloudBlobContainerWrapper getContainerReference(String name)
return container;
}
+ @Override
+ public CloudBlobContainerWrapper getContainerReference(final String name,
+ final AccessTokenProvider tokenProvider)
+ throws URISyntaxException, StorageException {
+ return getContainerReference(name);
+ }
+
class MockCloudBlobContainerWrapper extends CloudBlobContainerWrapper {
private boolean created = false;
private HashMap metadata;
@@ -234,6 +242,13 @@ public CloudBlobDirectoryWrapper getDirectoryReference(String relativePath)
relativePath, true)));
}
+ @Override
+ public CloudBlobDirectoryWrapper getDirectoryReference(String relativePath, AccessTokenProvider tokenProvider)
+ throws URISyntaxException, StorageException {
+ return new MockCloudBlobDirectoryWrapper(new URI(fullUriString(
+ relativePath, true)));
+ }
+
@Override
public CloudBlockBlobWrapper getBlockBlobReference(String relativePath)
throws URISyntaxException, StorageException {
@@ -241,12 +256,27 @@ public CloudBlockBlobWrapper getBlockBlobReference(String relativePath)
false)), null, 0);
}
+ @Override
+ public CloudBlobWrapper getBlockBlobReference(final String relativePath,
+ final AccessTokenProvider tokenProvider)
+ throws URISyntaxException, StorageException, IOException {
+ return new MockCloudBlockBlobWrapper(new URI(fullUriString(relativePath,
+ false)), null, 0);
+ }
+
@Override
public CloudPageBlobWrapper getPageBlobReference(String blobAddressUri)
throws URISyntaxException, StorageException {
return new MockCloudPageBlobWrapper(new URI(blobAddressUri), null, 0);
}
+ @Override
+ public CloudBlobWrapper getPageBlobReference(String blobAddressUri,
+ final AccessTokenProvider tokenProvider)
+ throws URISyntaxException, StorageException, IOException {
+ return new MockCloudPageBlobWrapper(new URI(blobAddressUri), null, 0);
+ }
+
// helper to create full URIs for directory and blob.
// use withTrailingSlash=true to get a good path for a directory.
private String fullUriString(String relativePath, boolean withTrailingSlash) {
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java
index 8ac36c299b65b5..a4dce44bde4055 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java
@@ -1689,7 +1689,7 @@ public void run() {
firstEndTime = System.currentTimeMillis();
lease.free();
LOG.info(name + " freed lease " + lease.getLeaseID());
- } catch (StorageException e) {
+ } catch (StorageException | IOException e) {
fail("Unanticipated exception");
}
} else if (name.equals("second-thread")) {
@@ -1712,7 +1712,7 @@ public void run() {
try {
lease.free();
LOG.info(name + " freed lease " + lease.getLeaseID());
- } catch (StorageException e) {
+ } catch (StorageException | IOException e) {
assertTrue("Unanticipated exception", false);
}
} else {
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
index fd2f2690daea21..b79e49390ac46f 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
@@ -25,6 +25,9 @@
import java.util.UUID;
import java.util.concurrent.Callable;
+import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider;
+import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -38,8 +41,10 @@
import org.apache.hadoop.fs.azurebfs.constants.FSOperationType;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
import org.apache.hadoop.fs.azurebfs.security.AbfsDelegationTokenManager;
+import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
+import org.apache.hadoop.fs.azurebfs.services.ITestAbfsClient;
import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore;
import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation;
@@ -55,6 +60,8 @@
import static org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.WASB_ACCOUNT_NAME_DOMAIN_SUFFIX;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.*;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_DNS_PREFIX;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.WASB_DNS_PREFIX;
import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.FILE_SYSTEM_NOT_FOUND;
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.*;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@@ -69,7 +76,7 @@ public abstract class AbstractAbfsIntegrationTest extends
AbstractAbfsTestWithTimeout {
private static final Logger LOG =
- LoggerFactory.getLogger(AbstractAbfsIntegrationTest.class);
+ LoggerFactory.getLogger(AbstractAbfsIntegrationTest.class);
private boolean isIPAddress;
private NativeAzureFileSystem wasb;
@@ -106,7 +113,7 @@ protected AbstractAbfsIntegrationTest() throws Exception {
if (authType == AuthType.SharedKey) {
assumeTrue("Not set: " + FS_AZURE_ACCOUNT_KEY,
- abfsConfig.get(FS_AZURE_ACCOUNT_KEY) != null);
+ abfsConfig.get(FS_AZURE_ACCOUNT_KEY) != null);
// Update credentials
} else {
assumeTrue("Not set: " + FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME,
@@ -239,6 +246,9 @@ public Hashtable call() throws Exception {
}
}
+ public AccessTokenProvider getAccessTokenProvider(final AzureBlobFileSystem fs) {
+ return ITestAbfsClient.getAccessTokenProvider(fs.getAbfsStore().getClient());
+ }
public void loadConfiguredFileSystem() throws Exception {
// disable auto-creation of filesystem
@@ -412,8 +422,7 @@ private static String convertTestUrls(
}
if (data != null) {
- data = data.replace("." + fromDnsPrefix + ".",
- "." + toDnsPrefix + ".");
+ data = data.replace(fromDnsPrefix, toDnsPrefix);
}
return data;
}
@@ -427,6 +436,24 @@ public AzureBlobFileSystemStore getAbfsStore(final AzureBlobFileSystem fs) {
return fs.getAbfsStore();
}
+ public PrefixMode getPrefixMode(final AzureBlobFileSystem fs) {
+ return fs.getAbfsStore().getAbfsConfiguration().getPrefixMode();
+ }
+
+ public AbfsClient getClient(final AzureBlobFileSystem fs) {
+ return fs.getAbfsStore().getClient();
+ }
+
+ public boolean isNamespaceEnabled(final AzureBlobFileSystem fs) throws AzureBlobFileSystemException {
+ return fs.getAbfsStore()
+ .getIsNamespaceEnabled(getTestTracingContext(fs, true));
+ }
+
+ public void setAbfsClient(AzureBlobFileSystemStore abfsStore,
+ AbfsClient client) {
+ abfsStore.setClient(client);
+ }
+
public Path makeQualified(Path path) throws java.io.IOException {
return getFileSystem().makeQualified(path);
}
@@ -469,7 +496,7 @@ protected AbfsOutputStream createAbfsOutputStreamWithFlushEnabled(
return (AbfsOutputStream) abfss.createFile(path, fs.getFsStatistics(),
true, FsPermission.getDefault(), FsPermission.getUMask(fs.getConf()),
- getTestTracingContext(fs, false));
+ getTestTracingContext(fs, false), null);
}
/**
@@ -486,4 +513,30 @@ protected long assertAbfsStatistics(AbfsStatistic statistic,
(long) metricMap.get(statistic.getStatName()));
return expectedValue;
}
+
+ /**
+ * For creating directory with implicit parents. Doesn't change already explicit
+ * parents.
+ */
+ void createAzCopyDirectory(Path path) throws Exception {
+ AzcopyHelper azcopyHelper = new AzcopyHelper(
+ getAccountName(), getFileSystemName(), getFileSystem().getAbfsStore()
+ .getAbfsConfiguration()
+ .getRawConfiguration(), getFileSystem().getAbfsStore().getPrefixMode());
+ azcopyHelper.createFolderUsingAzcopy(
+ getFileSystem().makeQualified(path).toUri().getPath().substring(1));
+ }
+
+ /**
+ * For creating files with implicit parents. Doesn't change already explicit
+ * parents.
+ */
+ void createAzCopyFile(Path path) throws Exception {
+ AzcopyHelper azcopyHelper = new AzcopyHelper(getAccountName(),
+ getFileSystemName(), getFileSystem().getAbfsStore()
+ .getAbfsConfiguration()
+ .getRawConfiguration(), getFileSystem().getAbfsStore().getPrefixMode());
+ azcopyHelper.createFileUsingAzcopy(
+ getFileSystem().makeQualified(path).toUri().getPath().substring(1));
+ }
}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AzcopyHelper.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AzcopyHelper.java
new file mode 100644
index 00000000000000..c15d094c83c907
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AzcopyHelper.java
@@ -0,0 +1,217 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileWriter;
+import java.io.IOException;
+
+import org.junit.Assume;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
+
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.FORWARD_SLASH;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_FIXED_TOKEN;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_DNS_PREFIX;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.WASB_DNS_PREFIX;
+
+public class AzcopyHelper {
+
+ File hadoopAzureDir;
+ String azcopyDirPath;
+ private String accountName;
+ private String fileSystemName;
+ private Configuration configuration;
+ private PrefixMode mode;
+
+ public AzcopyHelper(String accountName, String fileSystemName, Configuration configuration ,PrefixMode mode) throws Exception {
+ this.accountName = accountName.replace(ABFS_DNS_PREFIX, WASB_DNS_PREFIX);
+ this.fileSystemName = fileSystemName;
+ this.configuration = configuration;
+ this.mode = mode;
+ }
+
+ public void downloadAzcopyExecutableIfNotPresent() throws IOException, InterruptedException {
+ // Skip execution if Prefix Mode is DFS.
+ Assume.assumeTrue(mode == PrefixMode.BLOB);
+
+ // Find the hadoop-azure directory from the current working directory
+ File currentDir = new File(System.getProperty("user.dir"));
+ if (!currentDir.isDirectory() && !currentDir.getName().equals("hadoop-azure")) {
+ hadoopAzureDir = findHadoopAzureDir(currentDir);
+ if (hadoopAzureDir == null) {
+ throw new FileNotFoundException("hadoop-azure directory not found");
+ }
+ } else {
+ hadoopAzureDir = currentDir;
+ }
+
+ // Check if azcopy directory is present in the hadoop-azure directory, create it if it doesn't exist
+ azcopyDirPath = hadoopAzureDir.getAbsolutePath() + "/azcopy";
+ File azcopyDir = new File(azcopyDirPath);
+ if (!azcopyDir.exists()) {
+ boolean created = azcopyDir.mkdir();
+ // Check if azcopy is present in the azcopy directory
+ String azcopyPath = azcopyDirPath + "/azcopy";
+ File azcopyFile = new File(azcopyPath);
+ if (!azcopyFile.exists()) {
+ // If azcopy is not present, download and extract it
+ String downloadUrl = "https://aka.ms/downloadazcopy-v10-linux";
+ String downloadCmd = "wget " + downloadUrl + " -O azcopy.tar.gz" + " --no-check-certificate";
+ String[] downloadCmdArr = {"bash", "-c", downloadCmd};
+ Process downloadProcess = Runtime.getRuntime().exec(downloadCmdArr);
+ downloadProcess.waitFor();
+
+ // Extract the azcopy executable from the tarball
+ String extractCmd = "tar -xf azcopy.tar.gz -C " + hadoopAzureDir.getAbsolutePath();
+ String[] extractCmdArr = {"bash", "-c", extractCmd};
+ Process extractProcess = Runtime.getRuntime().exec(extractCmdArr);
+ extractProcess.waitFor();
+
+ // Rename the azcopy_linux_amd64_* directory to 'azcopy' and move it to the hadoop-azure directory
+ String renameCmd = "mv " + hadoopAzureDir.getAbsolutePath() + "/azcopy_linux_amd64_*/* " + azcopyDirPath;
+ String[] renameCmdArr = {"bash", "-c", renameCmd};
+ Process renameProcess = Runtime.getRuntime().exec(renameCmdArr);
+ renameProcess.waitFor();
+
+ // Remove the downloaded tarball and azcopy folder
+ String cleanupCmd = "rm -rf " + hadoopAzureDir.getAbsolutePath() + "/azcopy_linux_amd64_* azcopy.tar.gz";
+ String[] cleanupCmdArr = {"bash", "-c", cleanupCmd};
+ Process cleanupProcess = Runtime.getRuntime().exec(cleanupCmdArr);
+ cleanupProcess.waitFor();
+
+ // Set the execute permission on the azcopy executable
+ String chmodCmd = "chmod +x " + azcopyDirPath;
+ String[] chmodCmdArr = {"bash", "-c", chmodCmd};
+ Process chmodProcess = Runtime.getRuntime().exec(chmodCmdArr);
+ chmodProcess.waitFor();
+ }
+ }
+ // Change working directory to the hadoop-azure directory
+ System.setProperty("user.dir", hadoopAzureDir.getAbsolutePath());
+ }
+
+ private File findHadoopAzureDir(File dir) {
+ if (dir == null) {
+ return null;
+ }
+ File[] files = dir.listFiles();
+ if (files == null) {
+ return null;
+ }
+ for (File file : files) {
+ if (file.isDirectory() && file.getName().equals("hadoop-azure")) {
+ return file;
+ } else {
+ File hadoopAzureDir = findHadoopAzureDir(file);
+ if (hadoopAzureDir != null) {
+ return hadoopAzureDir;
+ }
+ }
+ }
+ return null;
+ }
+
+ public void createFileOrFolder(String pathFromContainerRoot, boolean isFile) throws Exception {
+ downloadAzcopyExecutableIfNotPresent();
+ String url = "https://" + accountName + FORWARD_SLASH + fileSystemName + FORWARD_SLASH +
+ pathFromContainerRoot;
+ // Add the SAS token in config file (should be Account SAS or Container SAS").
+ String configuredFixedToken = configuration.get(FS_AZURE_SAS_FIXED_TOKEN, null);
+ if (configuredFixedToken != null) {
+ if (isFile) {
+ createFileCreationScript(azcopyDirPath, "createFile" + Thread.currentThread().getName() + ".sh", azcopyDirPath, configuredFixedToken, url);
+ } else {
+ createFolderCreationScript(azcopyDirPath, "createFolder" + Thread.currentThread().getName() + ".sh", azcopyDirPath, configuredFixedToken, url);
+ }
+ } else {
+ throw new Exception("The SAS token provided is null");
+ }
+ String path;
+ if (isFile) {
+ path = azcopyDirPath + "/createFile" + Thread.currentThread().getName() + ".sh";
+ } else {
+ path = azcopyDirPath + "/createFolder" + Thread.currentThread().getName() + ".sh";
+ }
+ try {
+ ProcessBuilder pb = new ProcessBuilder(path);
+ Process p = pb.start();
+ // wait for the process to finish
+ int exitCode = p.waitFor();
+ } catch (IOException e) {
+ throw new IOException(e.getMessage());
+ } catch (InterruptedException e) {
+ throw new InterruptedException(e.getMessage());
+ }
+ String cleanupCmd = "rm -rf " + path;
+ String[] cleanupCmdArr = {"bash", "-c", cleanupCmd};
+ Process cleanupProcess = Runtime.getRuntime().exec(cleanupCmdArr);
+ cleanupProcess.waitFor();
+ }
+
+ public void createFileUsingAzcopy(String pathFromContainerRoot) throws Exception {
+ // Add the path you want to copy to as config.
+ if (pathFromContainerRoot != null) {
+ createFileOrFolder(pathFromContainerRoot, true);
+ }
+ }
+
+ public void createFolderUsingAzcopy(String pathFromContainerRoot) throws Exception {
+ // Add the path you want to copy to as config.
+ if (pathFromContainerRoot != null) {
+ createFileOrFolder(pathFromContainerRoot, false);
+ }
+ }
+
+ public static void createFileCreationScript(String folderPath, String scriptName, String azcopyPath, String sasToken, String containerName) {
+ String blobPath = containerName + "?" + sasToken; // construct the blob path
+ String scriptContent = "blobPath=\"" + blobPath + "\"\n"
+ + "echo $blobPath\n"
+ + azcopyPath + "/azcopy copy \"" + azcopyPath + "/NOTICE.txt\" $blobPath\n"; // construct the script content
+ File scriptFile = new File(folderPath, scriptName);
+ try {
+ FileWriter writer = new FileWriter(scriptFile);
+ writer.write(scriptContent);
+ writer.close();
+ boolean written = scriptFile.setExecutable(true); // make the script executable
+ System.out.println("Script created at " + scriptFile.getAbsolutePath());
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ public static void createFolderCreationScript(String folderPath, String scriptName, String azcopyPath, String sasToken, String containerName) {
+ String blobPath = containerName + "?" + sasToken; // construct the blob path
+ String scriptContent = "blobPath=\"" + blobPath + "\"\n"
+ + "echo $blobPath\n"
+ + azcopyPath + "/azcopy copy \"" + azcopyPath + "\" $blobPath --recursive\n"; // construct the script content
+ File scriptFile = new File(folderPath, scriptName);
+ try {
+ FileWriter writer = new FileWriter(scriptFile);
+ writer.write(scriptContent);
+ writer.close();
+ boolean written = scriptFile.setExecutable(true); // make the script executable
+ System.out.println("Script created at " + scriptFile.getAbsolutePath());
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/BlobDirectoryStateHelper.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/BlobDirectoryStateHelper.java
new file mode 100644
index 00000000000000..6e174b36046cde
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/BlobDirectoryStateHelper.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.util.List;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
+import org.apache.hadoop.fs.azurebfs.services.BlobProperty;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
+import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
+
+public class BlobDirectoryStateHelper {
+
+ /**
+ * To assert that a path exists as implicit directory we need two things to assert.
+ * 1. List blobs on the path should return some entries.
+ * 2. GetBlobProperties on path should fail.
+ * @param path to be checked
+ * @param fs AzureBlobFileSystem for API calls
+ * @return boolean whether the path exists as Implicit directory or not
+ */
+ public static boolean isImplicitDirectory(Path path, AzureBlobFileSystem fs, TracingContext testTracingContext) throws Exception {
+ path = new Path(fs.makeQualified(path).toUri().getPath());
+ if (fs.getAbfsStore().getPrefixMode() == PrefixMode.BLOB) {
+ List blobProperties = fs.getAbfsStore()
+ .getListBlobs(path,null, null, testTracingContext, 2, true);
+ if (blobProperties.size() == 0) {
+ return false;
+ }
+ try {
+ fs.getAbfsStore().getBlobProperty(
+ path, testTracingContext);
+ }
+ catch (AbfsRestOperationException ex) {
+ if (ex.getStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) {
+ return true;
+ }
+ }
+ return false;
+ }
+ else {
+ FileStatus[] statuses = fs.getAbfsStore()
+ .listStatus(path, testTracingContext);
+ if (statuses.length == 0) {
+ return false;
+ }
+ try {
+ FileStatus status = fs.getAbfsStore().getFileStatus(
+ path, testTracingContext, true);
+ return !status.isDirectory();
+ }
+ catch (AbfsRestOperationException ex) {
+ if (ex.getStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * To assert that a path exists as explicit directory
+ * For PrefixMode Blob: GetBlobProperties on path should succeed and marker should be present
+ * For PrefixMode DFS: GetFileStatus on path should succeed and marker should be present
+ * For Root on Blob: GetContainerProperty
+ * @param path to be checked
+ * @param fs AzureBlobFileSystem for API calls
+ * @return boolean whether the path exists as Implicit directory or not
+ */
+ public static boolean isExplicitDirectory(Path path, AzureBlobFileSystem fs, TracingContext testTracingContext) {
+ path = new Path(fs.makeQualified(path).toUri().getPath());
+
+ if (fs.getAbfsStore().getPrefixMode() == PrefixMode.DFS) {
+ FileStatus status;
+ try {
+ status = fs.getAbfsStore()
+ .getFileStatus(path, testTracingContext, false);
+ }
+ catch (IOException ex) {
+ return false;
+ }
+ return status.isDirectory();
+ }
+
+ else if (path.isRoot()) {
+ BlobProperty prop;
+ try {
+ prop = fs.getAbfsStore().getContainerProperty(testTracingContext);
+ }
+ catch(AzureBlobFileSystemException ex) {
+ return false;
+ }
+ return prop.getIsDirectory();
+ }
+
+ else {
+ BlobProperty prop;
+ try {
+ prop = fs.getAbfsStore().getBlobProperty(
+ path, testTracingContext);
+ }
+ catch(AzureBlobFileSystemException ex) {
+ return false;
+ }
+ return prop.getIsDirectory();
+ }
+ }
+}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsDurationTrackers.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsDurationTrackers.java
index 0997b3dbd44d4b..afd91824e38e9e 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsDurationTrackers.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsDurationTrackers.java
@@ -101,11 +101,16 @@ public void testAbfsHttpCallsDurations() throws IOException {
*/
private void assertDurationTracker(IOStatistics ioStatistics) {
for (AbfsStatistic abfsStatistic : HTTP_DURATION_TRACKER_LIST) {
- Assertions.assertThat(lookupMeanStatistic(ioStatistics,
- abfsStatistic.getStatName() + StoreStatisticNames.SUFFIX_MEAN).mean())
- .describedAs("The DurationTracker Named " + abfsStatistic.getStatName()
- + " Doesn't match the expected value.")
- .isGreaterThan(0.0);
+ // Since delete calls go to wasb endpoint and it doesn't have implementation for abfsStatistics
+ // we are avoiding incrementing stats for it. Can be reverted when we have blob implementation for
+ // delete.
+ if (!HTTP_DELETE_REQUEST.getStatName().equals(abfsStatistic.getStatName())) {
+ Assertions.assertThat(lookupMeanStatistic(ioStatistics,
+ abfsStatistic.getStatName() + StoreStatisticNames.SUFFIX_MEAN).mean())
+ .describedAs("The DurationTracker Named " + abfsStatistic.getStatName()
+ + " Doesn't match the expected value.")
+ .isGreaterThan(0.0);
+ }
}
}
}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsListStatusRemoteIterator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsListStatusRemoteIterator.java
index e2973912e23234..76e12b37f27455 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsListStatusRemoteIterator.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsListStatusRemoteIterator.java
@@ -33,13 +33,18 @@
import org.junit.Test;
import org.mockito.Mockito;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.azurebfs.services.AbfsListStatusRemoteIterator;
import org.apache.hadoop.fs.azurebfs.services.ListingSupport;
import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_DNS_PREFIX;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.WASB_DNS_PREFIX;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyList;
@@ -344,4 +349,66 @@ private List createFilesUnderDirectory(int numFiles, Path rootPath,
return fileNames;
}
+
+ @Test
+ public void testListStatusIteratorReturnStatusWithPathWithSameUriGivenInConfig()
+ throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ String accountName = getAccountName();
+ Boolean isAccountNameInDfs = accountName.contains(ABFS_DNS_PREFIX);
+ String dnsAssertion;
+ if (isAccountNameInDfs) {
+ dnsAssertion = ABFS_DNS_PREFIX;
+ } else {
+ dnsAssertion = WASB_DNS_PREFIX;
+ }
+
+ final Path path = new Path("/testDir/file");
+ fs.create(path);
+ assertListStatusIteratorPath(fs, accountName, dnsAssertion, path);
+
+ final Configuration configuration;
+ if (isAccountNameInDfs) {
+ configuration = new Configuration(getRawConfiguration());
+ configuration.set(FS_DEFAULT_NAME_KEY,
+ configuration.get(FS_DEFAULT_NAME_KEY)
+ .replace(ABFS_DNS_PREFIX, WASB_DNS_PREFIX));
+ dnsAssertion = WASB_DNS_PREFIX;
+
+ } else {
+ configuration = new Configuration(getRawConfiguration());
+ configuration.set(FS_DEFAULT_NAME_KEY,
+ configuration.get(FS_DEFAULT_NAME_KEY)
+ .replace(WASB_DNS_PREFIX, ABFS_DNS_PREFIX));
+ dnsAssertion = ABFS_DNS_PREFIX;
+ }
+ fs = (AzureBlobFileSystem) FileSystem.newInstance(configuration);
+ assertListStatusIteratorPath(fs, accountName, dnsAssertion, path);
+ }
+
+ private void assertListStatusIteratorPath(final AzureBlobFileSystem fs,
+ final String accountName,
+ final String dnsAssertion,
+ final Path path) throws IOException {
+ RemoteIterator fileStatusRemoteIterator = fs.listStatusIterator(
+ new Path(
+ "abfs://" + fs.getAbfsClient().getFileSystem() + "@" + accountName
+ + path.getParent().toUri().getPath()));
+
+ while (fileStatusRemoteIterator.hasNext()) {
+ FileStatus fileStatus = fileStatusRemoteIterator.next();
+ Assertions.assertThat(fileStatus.getPath().toString())
+ .contains(dnsAssertion);
+ }
+
+ fileStatusRemoteIterator = fs.listStatusIterator(new Path(
+ "abfs://" + fs.getAbfsClient().getFileSystem() + "@" + accountName
+ + path.toUri().getPath()));
+
+ while (fileStatusRemoteIterator.hasNext()) {
+ FileStatus fileStatus = fileStatusRemoteIterator.next();
+ Assertions.assertThat(fileStatus.getPath().toString())
+ .contains(dnsAssertion);
+ }
+ }
}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java
index d871befa430058..c93561005db7e5 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java
@@ -19,15 +19,22 @@
package org.apache.hadoop.fs.azurebfs;
import java.io.IOException;
+import java.lang.reflect.Field;
+import java.net.HttpURLConnection;
import java.util.Date;
-
import org.junit.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider;
+import org.apache.hadoop.fs.azurebfs.oauth2.AzureADAuthenticator;
import org.apache.hadoop.fs.azurebfs.oauth2.AzureADToken;
import org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider;
+import org.apache.hadoop.fs.azurebfs.services.ExponentialRetryPolicy;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_ATTEMPTS;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.junit.Assume.assumeThat;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.not;
@@ -40,6 +47,7 @@
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_MSI_TENANT;
+import static org.mockito.Mockito.times;
/**
* Test MsiTokenProvider.
@@ -47,6 +55,8 @@
public final class ITestAbfsMsiTokenProvider
extends AbstractAbfsIntegrationTest {
+ private static final int HTTP_TOO_MANY_REQUESTS = 429;
+
public ITestAbfsMsiTokenProvider() throws Exception {
super();
}
@@ -90,4 +100,109 @@ private String getTrimmedPasswordString(AbfsConfiguration conf, String key,
return value.trim();
}
+ /**
+ * Test to verify that token fetch is retried for throttling errors (too many requests 429).
+ * @throws Exception
+ */
+ @Test
+ public void testRetryForThrottling() throws Exception {
+ AbfsConfiguration conf = getConfiguration();
+
+ // Exception to be thrown with throttling error code 429.
+ AzureADAuthenticator.HttpException httpException
+ = new AzureADAuthenticator.HttpException(HTTP_TOO_MANY_REQUESTS,
+ "abc", "abc", "abc", "abc", "abc");
+
+ String tenantGuid = "abcd";
+ String clientId = "abcd";
+ String authEndpoint = getTrimmedPasswordString(conf,
+ FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT,
+ DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT);
+ String authority = getTrimmedPasswordString(conf,
+ FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY,
+ DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY);
+
+ // Mock the getTokenSingleCall to throw exception so the retry logic comes into place.
+ try (MockedStatic adAuthenticator = Mockito.mockStatic(
+ AzureADAuthenticator.class, Mockito.CALLS_REAL_METHODS)) {
+ adAuthenticator.when(
+ () -> AzureADAuthenticator.getTokenSingleCall(Mockito.anyString(),
+ Mockito.anyString(), Mockito.any(), Mockito.anyString(),
+ Mockito.anyBoolean())).thenThrow(httpException);
+
+ // Mock the tokenFetchRetryPolicy to verify retries.
+ ExponentialRetryPolicy exponentialRetryPolicy = Mockito.spy(
+ conf.getOauthTokenFetchRetryPolicy());
+ Field tokenFetchRetryPolicy = AzureADAuthenticator.class.getDeclaredField(
+ "tokenFetchRetryPolicy");
+ tokenFetchRetryPolicy.setAccessible(true);
+ tokenFetchRetryPolicy.set(ExponentialRetryPolicy.class,
+ exponentialRetryPolicy);
+
+ AccessTokenProvider tokenProvider = new MsiTokenProvider(authEndpoint,
+ tenantGuid, clientId, authority);
+ AzureADToken token = null;
+ intercept(AzureADAuthenticator.HttpException.class,
+ tokenProvider::getToken);
+
+ // If the status code doesn't qualify for retry shouldRetry returns false and the loop ends.
+ // It being called multiple times verifies that the retry was done for the throttling status code 429.
+ Mockito.verify(exponentialRetryPolicy,
+ times(DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_ATTEMPTS + 1))
+ .shouldRetry(Mockito.anyInt(), Mockito.anyInt());
+ }
+ }
+
+ /**
+ * Test to verify that token fetch is not retried for resource not found errors.
+ * @throws Exception
+ */
+ @Test
+ public void testNoRetryForResourceNotFound() throws Exception {
+ AbfsConfiguration conf = getConfiguration();
+
+ // Exception to be thrown 404 error code.
+ AzureADAuthenticator.HttpException httpException
+ = new AzureADAuthenticator.HttpException(HttpURLConnection.HTTP_NOT_FOUND,
+ "abc", "abc", "abc", "abc", "abc");
+
+ String tenantGuid = "abcd";
+ String clientId = "abcd";
+ String authEndpoint = getTrimmedPasswordString(conf,
+ FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT,
+ DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT);
+ String authority = getTrimmedPasswordString(conf,
+ FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY,
+ DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY);
+
+ // Mock the getTokenSingleCall to throw exception.
+ try (MockedStatic adAuthenticator = Mockito.mockStatic(
+ AzureADAuthenticator.class, Mockito.CALLS_REAL_METHODS)) {
+ adAuthenticator.when(
+ () -> AzureADAuthenticator.getTokenSingleCall(Mockito.anyString(),
+ Mockito.anyString(), Mockito.any(), Mockito.anyString(),
+ Mockito.anyBoolean())).thenThrow(httpException);
+
+ // Mock the tokenFetchRetryPolicy to verify no retries.
+ ExponentialRetryPolicy exponentialRetryPolicy = Mockito.spy(
+ conf.getOauthTokenFetchRetryPolicy());
+ Field tokenFetchRetryPolicy = AzureADAuthenticator.class.getDeclaredField(
+ "tokenFetchRetryPolicy");
+ tokenFetchRetryPolicy.setAccessible(true);
+ tokenFetchRetryPolicy.set(ExponentialRetryPolicy.class,
+ exponentialRetryPolicy);
+
+ AccessTokenProvider tokenProvider = new MsiTokenProvider(authEndpoint,
+ tenantGuid, clientId, authority);
+ AzureADToken token = null;
+ intercept(AzureADAuthenticator.HttpException.class,
+ tokenProvider::getToken);
+
+ // If the status code doesn't qualify for retry shouldRetry returns false and the loop ends.
+ // It being called only once verifies that retry doesn't come into place..
+ Mockito.verify(exponentialRetryPolicy,
+ times(1))
+ .shouldRetry(Mockito.anyInt(), Mockito.anyInt());
+ }
+ }
}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsNetworkStatistics.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsNetworkStatistics.java
index 66b8da89572a1e..e7bed75f93143f 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsNetworkStatistics.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsNetworkStatistics.java
@@ -21,6 +21,8 @@
import java.io.IOException;
import java.util.Map;
+import org.apache.hadoop.fs.azurebfs.services.OperativeEndpoint;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.junit.Test;
@@ -127,8 +129,11 @@ public void testAbfsHttpSendStatistics() throws IOException {
// Operation: AbfsOutputStream close.
// Network Stats calculation: 1 flush (with close) is send.
// 1 flush request = 1 connection and 1 send request
- expectedConnectionsMade++;
- expectedRequestsSent++;
+ if (OperativeEndpoint.isIngressEnabledOnDFS(getPrefixMode(fs), fs.getAbfsStore().getAbfsConfiguration())) {
+ // No network call made for flush without append in case of blob endpoint.
+ expectedConnectionsMade++;
+ expectedRequestsSent++;
+ }
// --------------------------------------------------------------------
// Operation: Re-create the file / create overwrite scenario
@@ -181,7 +186,6 @@ public void testAbfsHttpSendStatistics() throws IOException {
assertAbfsStatistics(SEND_REQUESTS, expectedRequestsSent, metricMap);
assertAbfsStatistics(AbfsStatistic.BYTES_SENT, expectedBytesSent, metricMap);
}
-
}
/**
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java
index f4f0f231037e18..beada775ae87b3 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java
@@ -32,11 +32,14 @@
import org.apache.hadoop.fs.azurebfs.services.AbfsInputStream;
import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream;
import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderValidator;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_INFO;
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.APPENDBLOB_MAX_WRITE_BUFFER_SIZE;
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_READ_BUFFER_SIZE;
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MAX_BUFFER_SIZE;
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MIN_BUFFER_SIZE;
+import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.logIOStatisticsAtLevel;
/**
* Test read, write and seek.
@@ -45,20 +48,29 @@
*/
@RunWith(Parameterized.class)
public class ITestAbfsReadWriteAndSeek extends AbstractAbfsScaleTest {
- private static final Path TEST_PATH = new Path("/testfile");
-
- @Parameterized.Parameters(name = "Size={0}")
+ private static final String TEST_PATH = "/testfile";
+
+ /**
+ * Parameterize on read buffer size and readahead.
+ * For test performance, a full x*y test matrix is not used.
+ * @return the test parameters
+ */
+ @Parameterized.Parameters(name = "Size={0}-readahead={1}")
public static Iterable sizes() {
- return Arrays.asList(new Object[][]{{MIN_BUFFER_SIZE},
- {DEFAULT_READ_BUFFER_SIZE},
- {APPENDBLOB_MAX_WRITE_BUFFER_SIZE},
- {MAX_BUFFER_SIZE}});
+ return Arrays.asList(new Object[][]{{MIN_BUFFER_SIZE, true},
+ {DEFAULT_READ_BUFFER_SIZE, false},
+ {DEFAULT_READ_BUFFER_SIZE, true},
+ {APPENDBLOB_MAX_WRITE_BUFFER_SIZE, false},
+ {MAX_BUFFER_SIZE, true}});
}
private final int size;
+ private final boolean readaheadEnabled;
- public ITestAbfsReadWriteAndSeek(final int size) throws Exception {
+ public ITestAbfsReadWriteAndSeek(final int size,
+ final boolean readaheadEnabled) throws Exception {
this.size = size;
+ this.readaheadEnabled = readaheadEnabled;
}
@Test
@@ -71,17 +83,25 @@ private void testReadWriteAndSeek(int bufferSize) throws Exception {
final AbfsConfiguration abfsConfiguration = fs.getAbfsStore().getAbfsConfiguration();
abfsConfiguration.setWriteBufferSize(bufferSize);
abfsConfiguration.setReadBufferSize(bufferSize);
+ abfsConfiguration.setReadAheadEnabled(readaheadEnabled);
final byte[] b = new byte[2 * bufferSize];
new Random().nextBytes(b);
- try (FSDataOutputStream stream = fs.create(TEST_PATH)) {
+ Path testPath = path(TEST_PATH);
+ FSDataOutputStream stream = fs.create(testPath);
+ try {
stream.write(b);
+ } finally{
+ stream.close();
}
+ logIOStatisticsAtLevel(LOG, IOSTATISTICS_LOGGING_LEVEL_INFO, stream);
final byte[] readBuffer = new byte[2 * bufferSize];
int result;
- try (FSDataInputStream inputStream = fs.open(TEST_PATH)) {
+ IOStatisticsSource statisticsSource = null;
+ try (FSDataInputStream inputStream = fs.open(testPath)) {
+ statisticsSource = inputStream;
((AbfsInputStream) inputStream.getWrappedStream()).registerListener(
new TracingHeaderValidator(abfsConfiguration.getClientCorrelationId(),
fs.getFileSystemId(), FSOperationType.READ, true, 0,
@@ -99,6 +119,8 @@ private void testReadWriteAndSeek(int bufferSize) throws Exception {
inputStream.seek(0);
result = inputStream.read(readBuffer, 0, bufferSize);
}
+ logIOStatisticsAtLevel(LOG, IOSTATISTICS_LOGGING_LEVEL_INFO, statisticsSource);
+
assertNotEquals("data read in final read()", -1, result);
assertArrayEquals(readBuffer, b);
}
@@ -109,30 +131,35 @@ public void testReadAheadRequestID() throws java.io.IOException {
final AbfsConfiguration abfsConfiguration = fs.getAbfsStore().getAbfsConfiguration();
int bufferSize = MIN_BUFFER_SIZE;
abfsConfiguration.setReadBufferSize(bufferSize);
+ abfsConfiguration.setReadAheadEnabled(readaheadEnabled);
final byte[] b = new byte[bufferSize * 10];
new Random().nextBytes(b);
- try (FSDataOutputStream stream = fs.create(TEST_PATH)) {
+ Path testPath = path(TEST_PATH);
+ try (FSDataOutputStream stream = fs.create(testPath)) {
((AbfsOutputStream) stream.getWrappedStream()).registerListener(
new TracingHeaderValidator(abfsConfiguration.getClientCorrelationId(),
fs.getFileSystemId(), FSOperationType.WRITE, false, 0,
((AbfsOutputStream) stream.getWrappedStream())
.getStreamID()));
stream.write(b);
+ logIOStatisticsAtLevel(LOG, IOSTATISTICS_LOGGING_LEVEL_INFO, stream);
}
+
final byte[] readBuffer = new byte[4 * bufferSize];
int result;
fs.registerListener(
new TracingHeaderValidator(abfsConfiguration.getClientCorrelationId(),
fs.getFileSystemId(), FSOperationType.OPEN, false, 0));
- try (FSDataInputStream inputStream = fs.open(TEST_PATH)) {
+ try (FSDataInputStream inputStream = fs.open(testPath)) {
((AbfsInputStream) inputStream.getWrappedStream()).registerListener(
new TracingHeaderValidator(abfsConfiguration.getClientCorrelationId(),
fs.getFileSystemId(), FSOperationType.READ, false, 0,
((AbfsInputStream) inputStream.getWrappedStream())
.getStreamID()));
result = inputStream.read(readBuffer, 0, bufferSize*4);
+ logIOStatisticsAtLevel(LOG, IOSTATISTICS_LOGGING_LEVEL_INFO, inputStream);
}
fs.registerListener(null);
}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
index a71e7bc815f759..43702812fa2b38 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
@@ -18,8 +18,11 @@
package org.apache.hadoop.fs.azurebfs;
+import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.Hashtable;
+import org.apache.hadoop.fs.azurebfs.services.OperativeEndpoint;
import org.assertj.core.api.Assertions;
import org.junit.Assert;
import org.junit.Test;
@@ -31,6 +34,10 @@
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
+import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
+
+import org.mockito.Mockito;
import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.DOT;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION;
@@ -45,8 +52,15 @@
public class ITestAbfsRestOperationException extends AbstractAbfsIntegrationTest{
private static final String RETRY_TEST_TOKEN_PROVIDER = "org.apache.hadoop.fs.azurebfs.oauth2.RetryTestTokenProvider";
+ boolean useBlobEndpoint;
public ITestAbfsRestOperationException() throws Exception {
- super();
+ super.setup();
+ AzureBlobFileSystemStore abfsStore = getAbfsStore(getFileSystem());
+ PrefixMode prefixMode = abfsStore.getPrefixMode();
+ AbfsConfiguration abfsConfiguration = abfsStore.getAbfsConfiguration();
+ useBlobEndpoint = !(OperativeEndpoint.isIngressEnabledOnDFS(prefixMode, abfsConfiguration) ||
+ OperativeEndpoint.isMkdirEnabledOnDFS(abfsConfiguration) ||
+ OperativeEndpoint.isReadEnabledOnDFS(abfsConfiguration));
}
@Test
@@ -60,12 +74,19 @@ public void testAbfsRestOperationExceptionFormat() throws IOException {
String errorMessage = ex.getLocalizedMessage();
String[] errorFields = errorMessage.split(",");
- Assert.assertEquals(4, errorFields.length);
+ // Expected Fields are: Message, StatusCode, Method, URL, ActivityId(rId)
+ Assert.assertEquals(5, errorFields.length);
// Check status message, status code, HTTP Request Type and URL.
- Assert.assertEquals("Operation failed: \"The specified path does not exist.\"", errorFields[0].trim());
+ if (useBlobEndpoint) {
+ Assert.assertEquals("Operation failed: \"The specified blob does not exist.\"", errorFields[0].trim());
+ }
+ else {
+ Assert.assertEquals("Operation failed: \"The specified path does not exist.\"", errorFields[0].trim());
+ }
Assert.assertEquals("404", errorFields[1].trim());
Assert.assertEquals("HEAD", errorFields[2].trim());
Assert.assertTrue(errorFields[3].trim().startsWith("http"));
+ Assert.assertTrue(errorFields[4].trim().startsWith("rId:"));
}
try {
@@ -74,17 +95,63 @@ public void testAbfsRestOperationExceptionFormat() throws IOException {
// verify its format
String errorMessage = ex.getLocalizedMessage();
String[] errorFields = errorMessage.split(",");
+ // Flow is different for listStatusIterator being enabled or not.
+ if (!getAbfsStore(fs).getAbfsConfiguration().enableAbfsListIterator()) {
+ Assert.assertEquals(7, errorFields.length);
+ // Check status message, status code, HTTP Request Type and URL.
+ if (useBlobEndpoint) {
+ Assert.assertEquals("Operation failed: \"The specified blob does not exist.\"", errorFields[0].trim());
+ }
+ else {
+ Assert.assertEquals("Operation failed: \"The specified path does not exist.\"", errorFields[0].trim());
+ }
+ Assert.assertEquals("404", errorFields[1].trim());
+ Assert.assertEquals("GET", errorFields[2].trim());
+ Assert.assertTrue(errorFields[3].trim().startsWith("http"));
+ Assert.assertTrue(errorFields[4].trim().startsWith("rId:"));
+ // Check storage error code and storage error message.
+ Assert.assertEquals("PathNotFound", errorFields[4].trim());
+ Assert.assertTrue(errorFields[5].contains("RequestId")
+ && errorFields[5].contains("Time"));
+ } else {
+ Assert.assertEquals(5, errorFields.length);
+ // Check status message, status code, HTTP Request Type and URL.
+ if (useBlobEndpoint) {
+ Assert.assertEquals("Operation failed: \"The specified blob does not exist.\"", errorFields[0].trim());
+ }
+ else {
+ Assert.assertEquals("Operation failed: \"The specified path does not exist.\"", errorFields[0].trim());
+ }
+ Assert.assertEquals("404", errorFields[1].trim());
+ Assert.assertEquals("HEAD", errorFields[2].trim());
+ Assert.assertTrue(errorFields[3].trim().startsWith("http"));
+ Assert.assertTrue(errorFields[4].trim().startsWith("rId:"));
+ }
+ }
+ // Check Exception Format For Put Method
+ try {
+ if (useBlobEndpoint) {
+ Hashtable metadata = new Hashtable<>();
+ metadata.put("hi", "hello");
+ fs.getAbfsStore().setBlobMetadata(fs.makeQualified(nonExistedFilePath1),
+ metadata, getTestTracingContext(fs, true));
+ }
+ } catch (AbfsRestOperationException ex) {
+ String errorMessage = ex.getLocalizedMessage();
+ String[] errorFields = errorMessage.split(",");
- Assert.assertEquals(6, errorFields.length);
+ Assert.assertEquals(7, errorFields.length);
// Check status message, status code, HTTP Request Type and URL.
- Assert.assertEquals("Operation failed: \"The specified path does not exist.\"", errorFields[0].trim());
+ if (useBlobEndpoint) {
+ Assert.assertEquals("Operation failed: \"The specified blob does not exist.\"", errorFields[0].trim());
+ }
+ else {
+ Assert.assertEquals("Operation failed: \"The specified path does not exist.\"", errorFields[0].trim());
+ }
Assert.assertEquals("404", errorFields[1].trim());
- Assert.assertEquals("GET", errorFields[2].trim());
+ Assert.assertEquals("PUT", errorFields[2].trim());
Assert.assertTrue(errorFields[3].trim().startsWith("http"));
- // Check storage error code and storage error message.
- Assert.assertEquals("PathNotFound", errorFields[4].trim());
- Assert.assertTrue(errorFields[5].contains("RequestId")
- && errorFields[5].contains("Time"));
+ Assert.assertTrue(errorFields[4].trim().startsWith("rId:"));
}
}
@@ -98,39 +165,43 @@ public void testCustomTokenFetchRetryCount() throws Exception {
public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exception {
AzureBlobFileSystem fs = this.getFileSystem();
- Configuration config = new Configuration(this.getRawConfiguration());
+ Configuration config = Mockito.spy(new Configuration(this.getRawConfiguration()));
String accountName = config.get("fs.azure.abfs.account.name");
// Setup to configure custom token provider
+ config.set("fs.azure.account.auth.type", "Custom");
config.set("fs.azure.account.auth.type." + accountName, "Custom");
+ config.set("fs.azure.account.oauth.provider.type", "org.apache.hadoop.fs"
+ + ".azurebfs.oauth2.RetryTestTokenProvider");
config.set("fs.azure.account.oauth.provider.type." + accountName, "org.apache.hadoop.fs"
+ ".azurebfs.oauth2.RetryTestTokenProvider");
config.set("fs.azure.custom.token.fetch.retry.count", Integer.toString(numOfRetries));
// Stop filesystem creation as it will lead to calls to store.
config.set("fs.azure.createRemoteFileSystemDuringInitialization", "false");
- final AzureBlobFileSystem fs1 =
- (AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(),
- config);
- RetryTestTokenProvider.ResetStatusToFirstTokenFetch();
-
intercept(Exception.class,
- ()-> {
- fs1.getFileStatus(new Path("/"));
- });
+ ()-> {
+ final AzureBlobFileSystem fs1 =
+ (AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(),
+ config);
+ });
// Number of retries done should be as configured
- Assert.assertTrue(
- "Number of token fetch retries (" + RetryTestTokenProvider.reTryCount
+ Assert.assertEquals("Number of token fetch retries (" + RetryTestTokenProvider.reTryCount
+ ") done, does not match with fs.azure.custom.token.fetch.retry.count configured (" + numOfRetries
- + ")", RetryTestTokenProvider.reTryCount == numOfRetries);
+ + ")", RetryTestTokenProvider.reTryCount, numOfRetries);
+
+ RetryTestTokenProvider.ResetStatusToFirstTokenFetch();
}
@Test
public void testAuthFailException() throws Exception {
- Configuration config = new Configuration(getRawConfiguration());
+ Configuration config = Mockito.spy(new Configuration(getRawConfiguration()));
String accountName = config
.get(FS_AZURE_ABFS_ACCOUNT_NAME);
// Setup to configure custom token provider
+ config.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, "Custom");
+ config.set(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME,
+ RETRY_TEST_TOKEN_PROVIDER);
config.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + DOT
+ accountName, "Custom");
config.set(
@@ -139,9 +210,8 @@ public void testAuthFailException() throws Exception {
// Stop filesystem creation as it will lead to calls to store.
config.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false");
- final AzureBlobFileSystem fs = getFileSystem(config);
try {
- fs.getFileStatus(new Path("/"));
+ final AzureBlobFileSystem fs = getFileSystem(config);
fail("Should fail at auth token fetch call");
} catch (AbfsRestOperationException e) {
String errorDesc = "Should throw RestOp exception on AAD failure";
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStatistics.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStatistics.java
index e6b572de9717c4..ea0d074b6a1fd7 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStatistics.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStatistics.java
@@ -21,6 +21,7 @@
import java.io.IOException;
import java.util.Map;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
import org.junit.Before;
import org.junit.Test;
@@ -102,9 +103,10 @@ public void testCreateStatistics() throws IOException {
assertAbfsStatistics(AbfsStatistic.CALL_CREATE, 1, metricMap);
assertAbfsStatistics(AbfsStatistic.CALL_CREATE_NON_RECURSIVE, 1, metricMap);
assertAbfsStatistics(AbfsStatistic.FILES_CREATED, 1, metricMap);
+ // Child calls mkdirs for parent in case of blob.
assertAbfsStatistics(AbfsStatistic.DIRECTORIES_CREATED, 1, metricMap);
assertAbfsStatistics(AbfsStatistic.CALL_MKDIRS, 1, metricMap);
- assertAbfsStatistics(AbfsStatistic.CALL_GET_FILE_STATUS, 2, metricMap);
+ assertAbfsStatistics(AbfsStatistic.CALL_GET_FILE_STATUS, 3, metricMap);
//re-initialising Abfs to reset statistic values.
fs.initialize(fs.getUri(), fs.getConf());
@@ -130,11 +132,12 @@ public void testCreateStatistics() throws IOException {
assertAbfsStatistics(AbfsStatistic.CALL_CREATE_NON_RECURSIVE, NUMBER_OF_OPS,
metricMap);
assertAbfsStatistics(AbfsStatistic.FILES_CREATED, NUMBER_OF_OPS, metricMap);
+ // Child calls mkdirs for parent in case of blob.
assertAbfsStatistics(AbfsStatistic.DIRECTORIES_CREATED, NUMBER_OF_OPS,
- metricMap);
+ metricMap);
assertAbfsStatistics(AbfsStatistic.CALL_MKDIRS, NUMBER_OF_OPS, metricMap);
assertAbfsStatistics(AbfsStatistic.CALL_GET_FILE_STATUS,
- 1 + NUMBER_OF_OPS, metricMap);
+ 1 + 2 * NUMBER_OF_OPS, metricMap);
}
/**
@@ -170,9 +173,16 @@ public void testDeleteStatistics() throws IOException {
since directory is delete recursively op_delete is called 2 times.
1 file is deleted, 1 listStatus() call is made.
*/
- assertAbfsStatistics(AbfsStatistic.CALL_DELETE, 2, metricMap);
- assertAbfsStatistics(AbfsStatistic.FILES_DELETED, 1, metricMap);
- assertAbfsStatistics(AbfsStatistic.CALL_LIST_STATUS, 1, metricMap);
+ // Calls go to wasb for now, will need to update this once goes to blob endpoint.
+ if (getPrefixMode(fs) == PrefixMode.BLOB) {
+ assertAbfsStatistics(AbfsStatistic.CALL_DELETE, 1, metricMap);
+ assertAbfsStatistics(AbfsStatistic.FILES_DELETED, 0, metricMap);
+ assertAbfsStatistics(AbfsStatistic.CALL_LIST_STATUS, 0, metricMap);
+ } else {
+ assertAbfsStatistics(AbfsStatistic.CALL_DELETE, 2, metricMap);
+ assertAbfsStatistics(AbfsStatistic.FILES_DELETED, 1, metricMap);
+ assertAbfsStatistics(AbfsStatistic.CALL_LIST_STATUS, 1, metricMap);
+ }
/*
creating a root directory and deleting it recursively to see if
@@ -184,7 +194,12 @@ public void testDeleteStatistics() throws IOException {
metricMap = fs.getInstrumentationMap();
//Test for directories_deleted.
- assertAbfsStatistics(AbfsStatistic.DIRECTORIES_DELETED, 1, metricMap);
+ // Calls go to wasb for now, will need to update this once goes to blob endpoint.
+ if (getPrefixMode(fs) == PrefixMode.BLOB) {
+ assertAbfsStatistics(AbfsStatistic.DIRECTORIES_DELETED, 0, metricMap);
+ } else {
+ assertAbfsStatistics(AbfsStatistic.DIRECTORIES_DELETED, 1, metricMap);
+ }
}
/**
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAppend.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAppend.java
index 4b1e497f64f84b..fee6a3c2d68d17 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAppend.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAppend.java
@@ -20,8 +20,30 @@
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
import java.util.Random;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
+import org.apache.hadoop.fs.azurebfs.contracts.services.AppendRequestParameters;
+import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
+import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream;
+import org.apache.hadoop.fs.azurebfs.services.OperativeEndpoint;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
+import org.apache.hadoop.fs.azurebfs.services.ITestAbfsClient;
+import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
+import org.junit.Assume;
import org.junit.Test;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -29,6 +51,20 @@
import org.apache.hadoop.fs.azurebfs.constants.FSOperationType;
import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderValidator;
import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
+
+import org.mockito.Mockito;
+
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_ENABLE_SMALL_WRITE_OPTIMIZATION;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_CONDITIONAL_CREATE_OVERWRITE;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_INFINITE_LEASE_KEY;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_LEASE_THREADS;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_MB;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.isNull;
/**
* Test append operations.
@@ -37,6 +73,7 @@ public class ITestAzureBlobFileSystemAppend extends
AbstractAbfsIntegrationTest {
private static final Path TEST_FILE_PATH = new Path("testfile");
private static final Path TEST_FOLDER_PATH = new Path("testFolder");
+ private static final String TEST_FILE = "testfile";
public ITestAzureBlobFileSystemAppend() throws Exception {
super();
@@ -68,7 +105,6 @@ public void testAppendFileAfterDelete() throws Exception {
final Path filePath = TEST_FILE_PATH;
ContractTestUtils.touch(fs, filePath);
fs.delete(filePath, false);
-
fs.append(filePath);
}
@@ -80,6 +116,108 @@ public void testAppendDirectory() throws Exception {
fs.append(folderPath);
}
+ @Test(expected = FileNotFoundException.class)
+ public void testAppendImplicitDirectory() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path folderPath = TEST_FOLDER_PATH;
+ fs.mkdirs(folderPath);
+ fs.append(folderPath.getParent());
+ }
+
+ @Test(expected = FileNotFoundException.class)
+ public void testAppendFileNotExists() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.append(TEST_FOLDER_PATH);
+ }
+
+ @Test(expected = IOException.class)
+ public void testIsAppendBlob() throws Exception {
+ final AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ Mockito.doReturn(true).when(store).isAppendBlobKey(anyString());
+ fs.getAbfsStore().getAbfsConfiguration().setPrefixMode(PrefixMode.BLOB);
+
+ // Set abfsStore as our mocked value.
+ Field privateField = AzureBlobFileSystem.class.getDeclaredField("abfsStore");
+ privateField.setAccessible(true);
+ privateField.set(fs, store);
+
+ fs.create(TEST_FILE_PATH);
+ }
+
+ @Test(expected = IOException.class)
+ public void testSmallWriteBlob() throws Exception {
+ Configuration configuration = getRawConfiguration();
+ configuration.set(AZURE_ENABLE_SMALL_WRITE_OPTIMIZATION, "true");
+ AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.newInstance(configuration);
+ fs.getAbfsStore().getAbfsConfiguration().setPrefixMode(PrefixMode.BLOB);
+
+ fs.create(TEST_FILE_PATH);
+ }
+
+ /** Create file over dfs endpoint and append over blob endpoint **/
+ @Test
+ public void testCreateOverDfsAppendOverBlob() throws IOException {
+ final AzureBlobFileSystem fs = getFileSystem();
+ Assume.assumeTrue(fs.getAbfsStore().getAbfsConfiguration().getPrefixMode() == PrefixMode.BLOB);
+ fs.getAbfsClient().createPath(makeQualified(TEST_FILE_PATH).toUri().getPath(), true, false,
+ null, null, false,
+ null, getTestTracingContext(fs, true));
+ FSDataOutputStream outputStream = fs.append(TEST_FILE_PATH);
+ outputStream.write(10);
+ outputStream.hsync();
+ outputStream.write(20);
+ outputStream.hsync();
+ outputStream.write(30);
+ outputStream.hsync();
+ }
+
+ /**
+ * Create directory over dfs endpoint and append over blob endpoint.
+ * Should return error as append is not supported for directory.
+ * **/
+ @Test(expected = IOException.class)
+ public void testCreateExplicitDirectoryOverDfsAppendOverBlob() throws IOException {
+ final AzureBlobFileSystem fs = getFileSystem();
+ Assume.assumeTrue(fs.getAbfsStore().getAbfsConfiguration().getPrefixMode() == PrefixMode.BLOB);
+ fs.getAbfsClient().createPath(makeQualified(TEST_FOLDER_PATH).toUri().getPath(), false, false,
+ null, null, false,
+ null, getTestTracingContext(fs, true));
+ FSDataOutputStream outputStream = fs.append(TEST_FOLDER_PATH);
+ outputStream.write(10);
+ outputStream.hsync();
+ }
+
+ /**
+ * Recreate file between append and flush. Etag mismatch happens.
+ **/
+ @Test(expected = IOException.class)
+ public void testRecreateAppendAndFlush() throws IOException {
+ final AzureBlobFileSystem fs = getFileSystem();
+ Assume.assumeTrue(!OperativeEndpoint.isIngressEnabledOnDFS(getPrefixMode(fs), getAbfsStore(fs).getAbfsConfiguration()));
+ fs.create(TEST_FILE_PATH);
+ FSDataOutputStream outputStream = fs.append(TEST_FILE_PATH);
+ outputStream.write(10);
+ final AzureBlobFileSystem fs1 = (AzureBlobFileSystem) FileSystem.newInstance(getRawConfiguration());
+ FSDataOutputStream outputStream1 = fs1.create(TEST_FILE_PATH);
+ outputStream.hsync();
+ }
+
+ /**
+ * Recreate file between append and flush using dfs. Etag mismatch happens.
+ **/
+ @Test(expected = IOException.class)
+ public void testRecreateDirectoryAppendAndFlush() throws IOException {
+ final AzureBlobFileSystem fs = getFileSystem();
+ Assume.assumeTrue(fs.getAbfsStore().getAbfsConfiguration().getPrefixMode() == PrefixMode.BLOB);
+ fs.create(TEST_FILE_PATH);
+ FSDataOutputStream outputStream = fs.append(TEST_FILE_PATH);
+ outputStream.write(10);
+ final AzureBlobFileSystem fs1 = (AzureBlobFileSystem) FileSystem.newInstance(getRawConfiguration());
+ fs1.mkdirs(TEST_FILE_PATH);
+ outputStream.hsync();
+ }
+
@Test
public void testTracingForAppend() throws IOException {
AzureBlobFileSystem fs = getFileSystem();
@@ -89,4 +227,507 @@ public void testTracingForAppend() throws IOException {
fs.getFileSystemId(), FSOperationType.APPEND, false, 0));
fs.append(TEST_FILE_PATH, 10);
}
+
+ /**
+ * Verify that no calls to getBlockList were made.
+ */
+ @Test
+ public void testCreateEmptyBlob() throws IOException {
+ Assume.assumeTrue(getFileSystem().getAbfsStore().getPrefixMode() == PrefixMode.BLOB);
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ AbfsClient client = store.getClient();
+ AbfsClient spiedClient = Mockito.spy(client);
+ store.setClient(spiedClient);
+
+ fs.create(TEST_FILE_PATH);
+ Mockito.verify(spiedClient, Mockito.times(0))
+ .getBlockList(Mockito.any(String.class),
+ Mockito.any(TracingContext.class));
+ }
+
+ @Test
+ public void testAppendImplicitDirectoryAzcopy() throws Exception {
+ Assume.assumeTrue(getFileSystem().getAbfsStore().getPrefixMode() == PrefixMode.BLOB);
+ AzureBlobFileSystem fs = getFileSystem();
+ createAzCopyDirectory(new Path("/src"));
+ createAzCopyFile(new Path("/src/file"));
+ intercept(AbfsRestOperationException.class, () -> {
+ fs.getAbfsStore().getBlobProperty(new Path("/src"), getTestTracingContext(fs, true));
+ });
+ intercept(FileNotFoundException.class, () -> fs.append(new Path("/src")));
+ }
+
+ /**
+ * Verify that no calls to getBlockList were made.
+ */
+ @Test
+ public void testCreateNonEmptyBlob() throws IOException {
+ Assume.assumeTrue(getFileSystem().getAbfsStore().getPrefixMode() == PrefixMode.BLOB);
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ AbfsClient client = store.getClient();
+ AbfsClient testClient = Mockito.spy(ITestAbfsClient.createTestClientFromCurrentContext(
+ client,
+ fs.getAbfsStore().getAbfsConfiguration()));
+ store.setClient(testClient);
+
+ FSDataOutputStream outputStream = fs.create(TEST_FILE_PATH);
+ outputStream.write(10);
+ outputStream.hsync();
+ outputStream.close();
+ fs.append(TEST_FILE_PATH);
+ Mockito.verify(testClient, Mockito.times(1))
+ .getBlockList(Mockito.any(String.class),
+ Mockito.any(TracingContext.class));
+ }
+
+ /**
+ * Verify that if getBlockList throws exception append should fail.
+ */
+ @Test
+ public void testValidateGetBlockList() throws Exception {
+ Assume.assumeTrue(getFileSystem().getAbfsStore().getPrefixMode() == PrefixMode.BLOB);
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ AbfsClient client = store.getClient();
+ AbfsClient testClient = Mockito.spy(ITestAbfsClient.createTestClientFromCurrentContext(
+ client,
+ fs.getAbfsStore().getAbfsConfiguration()));
+ store.setClient(testClient);
+
+ AzureBlobFileSystemException exception = Mockito.mock(AzureBlobFileSystemException.class);
+ // Throw exception when getBlockList is called
+ Mockito.doThrow(exception).when(testClient)
+ .getBlockList(Mockito.any(), Mockito.any(TracingContext.class));
+
+ // Create a non-empty file
+ FSDataOutputStream outputStream = fs.create(TEST_FILE_PATH);
+ outputStream.write(10);
+ outputStream.hsync();
+ outputStream.close();
+
+ intercept(AzureBlobFileSystemException.class, () -> fs.getAbfsStore()
+ .openFileForWrite(TEST_FILE_PATH, null, false, getTestTracingContext(fs, true)));
+ }
+
+ /**
+ * Verify that parallel write with same offset from different output streams will not throw exception.
+ **/
+ @Test
+ public void testParallelWriteSameOffsetDifferentOutputStreams() throws Exception {
+ Configuration configuration = getRawConfiguration();
+ configuration.set(FS_AZURE_ENABLE_CONDITIONAL_CREATE_OVERWRITE, "false");
+ AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.newInstance(configuration);
+ ExecutorService executorService = Executors.newFixedThreadPool(5);
+ List> futures = new ArrayList<>();
+
+ final byte[] b = new byte[8 * ONE_MB];
+ new Random().nextBytes(b);
+
+ // Create three output streams
+ FSDataOutputStream out1 = fs.create(TEST_FILE_PATH);
+ FSDataOutputStream out2 = fs.append(TEST_FILE_PATH);
+ FSDataOutputStream out3 = fs.append(TEST_FILE_PATH);
+
+ // Submit tasks to write to each output stream with the same offset
+ futures.add(executorService.submit(() -> {
+ try {
+ out1.write(b, 10, 200);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }));
+
+ futures.add(executorService.submit(() -> {
+ try {
+ out2.write(b, 10, 200);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }));
+
+ futures.add(executorService.submit(() -> {
+ try {
+ out3.write(b, 10, 200);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }));
+
+ int exceptionCaught = 0;
+ for (Future future : futures) {
+ try {
+ future.get(); // wait for the task to complete and handle any exceptions thrown by the lambda expression
+ } catch (ExecutionException e) {
+ Throwable cause = e.getCause();
+ if (cause instanceof RuntimeException) {
+ exceptionCaught++;
+ } else {
+ System.err.println("Unexpected exception caught: " + cause);
+ }
+ } catch (InterruptedException e) {
+ // handle interruption
+ }
+ }
+ assertEquals(exceptionCaught, 0);
+ }
+
+
+ /**
+ * Verify that parallel write for different content length will not throw exception.
+ **/
+ @Test
+ public void testParallelWriteDifferentContentLength() throws Exception {
+ Configuration configuration = getRawConfiguration();
+ configuration.set(FS_AZURE_ENABLE_CONDITIONAL_CREATE_OVERWRITE, "false");
+ FileSystem fs = FileSystem.newInstance(configuration);
+ ExecutorService executorService = Executors.newFixedThreadPool(5);
+ List> futures = new ArrayList<>();
+
+ // Create three output streams with different content length
+ FSDataOutputStream out1 = fs.create(TEST_FILE_PATH);
+ final byte[] b1 = new byte[8 * ONE_MB];
+ new Random().nextBytes(b1);
+
+ FSDataOutputStream out2 = fs.append(TEST_FILE_PATH);
+ FSDataOutputStream out3 = fs.append(TEST_FILE_PATH);
+
+ // Submit tasks to write to each output stream
+ futures.add(executorService.submit(() -> {
+ try {
+ out1.write(b1, 10, 200);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }));
+
+ futures.add(executorService.submit(() -> {
+ try {
+ out2.write(b1, 20, 300);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }));
+
+ futures.add(executorService.submit(() -> {
+ try {
+ out3.write(b1, 30, 400);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }));
+
+ int exceptionCaught = 0;
+ for (Future future : futures) {
+ try {
+ future.get(); // wait for the task to complete and handle any exceptions thrown by the lambda expression
+ } catch (ExecutionException e) {
+ Throwable cause = e.getCause();
+ if (cause instanceof RuntimeException) {
+ exceptionCaught++;
+ } else {
+ System.err.println("Unexpected exception caught: " + cause);
+ }
+ } catch (InterruptedException e) {
+ // handle interruption
+ }
+ }
+ assertEquals(exceptionCaught, 0);
+ }
+
+ /**
+ * Verify that parallel write for different content length will not throw exception.
+ **/
+ @Test
+ public void testParallelWriteOutputStreamClose() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ Assume.assumeTrue(!OperativeEndpoint.isIngressEnabledOnDFS(getPrefixMode(fs), getAbfsStore(fs).getAbfsConfiguration()));
+ final Path SECONDARY_FILE_PATH = new Path("secondarytestfile");
+ ExecutorService executorService = Executors.newFixedThreadPool(2);
+ List> futures = new ArrayList<>();
+
+ FSDataOutputStream out1 = fs.create(SECONDARY_FILE_PATH);
+ AbfsOutputStream outputStream1 = (AbfsOutputStream) out1.getWrappedStream();
+ String fileETag = outputStream1.getETag();
+ final byte[] b1 = new byte[8 * ONE_MB];
+ new Random().nextBytes(b1);
+ final byte[] b2 = new byte[8 * ONE_MB];
+ new Random().nextBytes(b2);
+
+ FSDataOutputStream out2 = fs.append(SECONDARY_FILE_PATH);
+
+ // Submit tasks to write to each output stream
+ futures.add(executorService.submit(() -> {
+ try {
+ out1.write(b1, 0, 200);
+ out1.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }));
+
+ futures.add(executorService.submit(() -> {
+ try {
+ out2.write(b2, 0, 400);
+ out2.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }));
+
+ int exceptionCaught = 0;
+
+ for (Future future : futures) {
+ try {
+ future.get(); // wait for the task to complete and handle any exceptions thrown by the lambda expression
+ } catch (ExecutionException e) {
+ Throwable cause = e.getCause();
+ if (cause instanceof RuntimeException) {
+ exceptionCaught++;
+ } else {
+ System.err.println("Unexpected exception caught: " + cause);
+ }
+ } catch (InterruptedException e) {
+ // handle interruption
+ }
+ }
+
+ assertEquals(exceptionCaught, 1);
+ // Validate that the data written in the buffer is the same as what was read
+ final byte[] readBuffer = new byte[8 * ONE_MB];
+ int result;
+ FSDataInputStream inputStream = fs.open(SECONDARY_FILE_PATH);
+ inputStream.seek(0);
+
+ AbfsOutputStream outputStream2 = (AbfsOutputStream) out1.getWrappedStream();
+ String out1Etag = outputStream2.getETag();
+
+ AbfsOutputStream outputStream3 = (AbfsOutputStream) out2.getWrappedStream();
+ String out2Etag = outputStream3.getETag();
+
+ if (!fileETag.equals(out1Etag)) {
+ result = inputStream.read(readBuffer, 0, 4 * ONE_MB);
+ assertEquals(result, 200); // Verify that the number of bytes read matches the number of bytes written
+ assertArrayEquals(Arrays.copyOfRange(readBuffer, 0, result), Arrays.copyOfRange(b1, 0, result)); // Verify that the data read matches the original data written
+ } else if (!fileETag.equals(out2Etag)) {
+ result = inputStream.read(readBuffer, 0, 4 * ONE_MB);
+ assertEquals(result, 400); // Verify that the number of bytes read matches the number of bytes written
+ assertArrayEquals(Arrays.copyOfRange(readBuffer, 0, result), Arrays.copyOfRange(b2, 0, result)); // Verify that the data read matches the original data written
+ } else {
+ fail("Neither out1 nor out2 was flushed successfully.");
+ }
+ }
+
+ /**
+ * Verify that once flushed etag changes.
+ **/
+ @Test
+ public void testEtagMismatch() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ Assume.assumeTrue(!OperativeEndpoint.isIngressEnabledOnDFS(getPrefixMode(fs), getAbfsStore(fs).getAbfsConfiguration()));
+ FSDataOutputStream out1 = fs.create(TEST_FILE_PATH);
+ FSDataOutputStream out2 = fs.create(TEST_FILE_PATH);
+
+ out2.write(10);
+ out2.hsync();
+ out1.write(10);
+ intercept(IOException.class, () -> out1.hsync());
+ }
+
+ /**
+ * Verify that for flush without append no network calls are made.
+ **/
+ @Test
+ public void testNoNetworkCallsForFlush() throws Exception {
+ Assume.assumeTrue(getFileSystem().getAbfsStore().getPrefixMode() == PrefixMode.BLOB);
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ AbfsClient client = store.getClient();
+ AbfsClient spiedClient = Mockito.spy(client);
+ store.setClient(spiedClient);
+
+ FSDataOutputStream outputStream = fs.create(TEST_FILE_PATH);
+ outputStream.hsync();
+ Mockito.verify(spiedClient, Mockito.times(0))
+ .append(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(),
+ Mockito.any(TracingContext.class), Mockito.any());
+ Mockito.verify(spiedClient, Mockito.times(0)).
+ flush(any(byte[].class), anyString(), anyBoolean(), isNull(), isNull(), any(),
+ any(TracingContext.class));
+ }
+
+ /**
+ * Verify that for flush without append no network calls are made.
+ **/
+ @Test
+ public void testNoNetworkCallsForSecondFlush() throws Exception {
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ Assume.assumeTrue(!OperativeEndpoint.isIngressEnabledOnDFS(getPrefixMode(fs), getAbfsStore(fs).getAbfsConfiguration()));
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ AbfsClient client = store.getClient();
+ AbfsClient spiedClient = Mockito.spy(client);
+ store.setClient(spiedClient);
+
+ FSDataOutputStream outputStream = fs.create(TEST_FILE_PATH);
+ outputStream.write(10);
+ outputStream.hsync();
+ outputStream.close();
+ Mockito.verify(spiedClient, Mockito.times(1))
+ .append(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(),
+ Mockito.any(TracingContext.class), Mockito.any());
+ Mockito.verify(spiedClient, Mockito.times(1)).
+ flush(any(byte[].class), anyString(), anyBoolean(), isNull(), isNull(), any(),
+ any(TracingContext.class));
+ }
+
+ private AzureBlobFileSystem getCustomFileSystem(Path infiniteLeaseDirs, int numLeaseThreads) throws Exception {
+ Configuration conf = getRawConfiguration();
+ conf.setBoolean(String.format("fs.%s.impl.disable.cache", getAbfsScheme()), true);
+ conf.set(FS_AZURE_INFINITE_LEASE_KEY, infiniteLeaseDirs.toUri().getPath());
+ conf.setInt(FS_AZURE_LEASE_THREADS, numLeaseThreads);
+ return getFileSystem(conf);
+ }
+
+ @Test
+ public void testAppendWithLease() throws Exception {
+ Assume.assumeTrue(getFileSystem().getAbfsStore().getPrefixMode() == PrefixMode.BLOB);
+ final Path testFilePath = new Path(path(methodName.getMethodName()), TEST_FILE);
+ final AzureBlobFileSystem fs = Mockito.spy(getCustomFileSystem(testFilePath.getParent(), 1));
+ AbfsOutputStream outputStream = (AbfsOutputStream) fs.getAbfsStore().createFile(testFilePath, null, true,
+ null, null, getTestTracingContext(fs, true), null);
+ outputStream.write(10);
+ outputStream.close();
+ assertNotNull(outputStream.getLeaseId());
+ }
+
+
+ /**
+ * If a write operation fails asynchronously, when the next write comes once failure is
+ * registered, that operation would fail with the exception caught on previous
+ * write operation.
+ * The next close, hsync, hflush would also fail for the last caught exception.
+ */
+ @Test
+ public void testIntermittentAppendFailureToBeReported() throws Exception {
+ AzureBlobFileSystem fs = Mockito.spy(
+ (AzureBlobFileSystem) FileSystem.newInstance(getRawConfiguration()));
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ AbfsClient spiedClient = Mockito.spy(store.getClient());
+ store.setClient(spiedClient);
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ Mockito.doThrow(
+ new AbfsRestOperationException(503, "", "", new Exception()))
+ .when(spiedClient)
+ .append(Mockito.anyString(), Mockito.anyString(),
+ Mockito.any(byte[].class), Mockito.any(
+ AppendRequestParameters.class), Mockito.nullable(String.class),
+ Mockito.any(TracingContext.class), Mockito.nullable(String.class));
+
+ Mockito.doThrow(
+ new AbfsRestOperationException(503, "", "", new Exception()))
+ .when(spiedClient)
+ .append(Mockito.anyString(), Mockito.any(byte[].class), Mockito.any(
+ AppendRequestParameters.class), Mockito.nullable(String.class),
+ Mockito.any(TracingContext.class));
+
+ byte[] bytes = new byte[1024 * 1024 * 8];
+ new Random().nextBytes(bytes);
+ LambdaTestUtils.intercept(IOException.class, () -> {
+ try (FSDataOutputStream os = fs.create(new Path("/test/file"))) {
+ os.write(bytes);
+ }
+ });
+
+ LambdaTestUtils.intercept(IOException.class, () -> {
+ FSDataOutputStream os = fs.create(new Path("/test/file"));
+ os.write(bytes);
+ os.close();
+ });
+
+ LambdaTestUtils.intercept(IOException.class, () -> {
+ FSDataOutputStream os = fs.create(new Path("/test/file"));
+ os.write(bytes);
+ os.hsync();
+ });
+
+ LambdaTestUtils.intercept(IOException.class, () -> {
+ FSDataOutputStream os = fs.create(new Path("/test/file"));
+ os.write(bytes);
+ os.hflush();
+ });
+
+ LambdaTestUtils.intercept(IOException.class, () -> {
+ AbfsOutputStream os = (AbfsOutputStream) fs.create(new Path("/test/file"))
+ .getWrappedStream();
+ os.write(bytes);
+ while (!os.areWriteOperationsTasksDone()) ;
+ os.write(bytes);
+ });
+ }
+
+ /**
+ * Test to check when async write takes time, the close, hsync, hflush method
+ * wait to get async ops completed and then flush. If async ops fail, the methods
+ * will throw exception.
+ */
+ @Test
+ public void testWriteAsyncOpFailedAfterCloseCalled() throws Exception {
+ AzureBlobFileSystem fs = Mockito.spy(
+ (AzureBlobFileSystem) FileSystem.newInstance(getRawConfiguration()));
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ AbfsClient spiedClient = Mockito.spy(store.getClient());
+ store.setClient(spiedClient);
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+
+ byte[] bytes = new byte[1024 * 1024 * 8];
+ new Random().nextBytes(bytes);
+
+ AtomicInteger count = new AtomicInteger(0);
+
+ Mockito.doAnswer(answer -> {
+ count.incrementAndGet();
+ while (count.get() < 2) ;
+ Thread.sleep(1000);
+ throw new AbfsRestOperationException(503, "", "", new Exception());
+ })
+ .when(spiedClient)
+ .append(Mockito.anyString(), Mockito.anyString(),
+ Mockito.any(byte[].class), Mockito.any(
+ AppendRequestParameters.class), Mockito.nullable(String.class),
+ Mockito.any(TracingContext.class), Mockito.nullable(String.class));
+
+ Mockito.doAnswer(answer -> {
+ count.incrementAndGet();
+ while (count.get() < 2) ;
+ Thread.sleep(1000);
+ throw new AbfsRestOperationException(503, "", "", new Exception());
+ })
+ .when(spiedClient)
+ .append(Mockito.anyString(), Mockito.any(byte[].class), Mockito.any(
+ AppendRequestParameters.class), Mockito.nullable(String.class),
+ Mockito.any(TracingContext.class));
+
+ FSDataOutputStream os = fs.create(new Path("/test/file"));
+ os.write(bytes);
+ os.write(bytes);
+ LambdaTestUtils.intercept(IOException.class, os::close);
+
+ count.set(0);
+ FSDataOutputStream os1 = fs.create(new Path("/test/file1"));
+ os1.write(bytes);
+ os1.write(bytes);
+ LambdaTestUtils.intercept(IOException.class, os1::hsync);
+
+ count.set(0);
+ FSDataOutputStream os2 = fs.create(new Path("/test/file2"));
+ os2.write(bytes);
+ os2.write(bytes);
+ LambdaTestUtils.intercept(IOException.class, os2::hflush);
+ }
}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAttributes.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAttributes.java
index beb7d0ebaaa8ea..07a8799f1c19f5 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAttributes.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAttributes.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.fs.azurebfs;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
import java.util.EnumSet;
import org.junit.Assume;
@@ -26,8 +28,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.XAttrSetFlag;
-import org.apache.hadoop.fs.azurebfs.constants.FSOperationType;
-import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderValidator;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@@ -42,70 +43,210 @@ public ITestAzureBlobFileSystemAttributes() throws Exception {
super();
}
+ /**
+ * Test GetXAttr() and SetXAttr() with Unicode Attribute Values.
+ * DFS does not support Unicode characters in user-defined metadata properties.
+ * Blob Endpoint supports Unicode encoded in UTF_8 character encoding.
+ * @throws Exception
+ */
@Test
- public void testSetGetXAttr() throws Exception {
+ public void testGetSetXAttr() throws Exception {
AzureBlobFileSystem fs = getFileSystem();
- AbfsConfiguration conf = fs.getAbfsStore().getAbfsConfiguration();
- Assume.assumeTrue(getIsNamespaceEnabled(fs));
-
- byte[] attributeValue1 = fs.getAbfsStore().encodeAttribute("hi");
- byte[] attributeValue2 = fs.getAbfsStore().encodeAttribute("ä½ å¥½");
- String attributeName1 = "user.asciiAttribute";
- String attributeName2 = "user.unicodeAttribute";
- Path testFile = path("setGetXAttr");
-
- // after creating a file, the xAttr should not be present
- touch(testFile);
- assertNull(fs.getXAttr(testFile, attributeName1));
-
- // after setting the xAttr on the file, the value should be retrievable
- fs.registerListener(
- new TracingHeaderValidator(conf.getClientCorrelationId(),
- fs.getFileSystemId(), FSOperationType.SET_ATTR, true, 0));
- fs.setXAttr(testFile, attributeName1, attributeValue1);
- fs.setListenerOperation(FSOperationType.GET_ATTR);
- assertArrayEquals(attributeValue1, fs.getXAttr(testFile, attributeName1));
- fs.registerListener(null);
-
- // after setting a second xAttr on the file, the first xAttr values should not be overwritten
- fs.setXAttr(testFile, attributeName2, attributeValue2);
- assertArrayEquals(attributeValue1, fs.getXAttr(testFile, attributeName1));
- assertArrayEquals(attributeValue2, fs.getXAttr(testFile, attributeName2));
+ final Path testPath = new Path("a/b");
+ fs.create(testPath);
+ testGetSetXAttrHelper(fs, testPath, testPath);
+ }
+
+ @Test
+ public void testGetSetXAttrOnRoot() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ // TODO: Support SetXAttr() on root on DFS endpoint
+ Assume.assumeTrue(fs.getAbfsStore().getPrefixMode() == PrefixMode.BLOB);
+ final Path filePath = new Path("a/b");
+ final Path testPath = new Path("/");
+ fs.create(filePath);
+ testGetSetXAttrHelper(fs, filePath, testPath);
+ }
+
+ @Test
+ public void testGetSetXAttrOnImplicitDir() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ final Path testPath = new Path("a/b");
+ AzcopyHelper azcopyHelper = new AzcopyHelper(
+ getAccountName(),
+ getFileSystemName(),
+ getRawConfiguration(),
+ fs.getAbfsStore().getPrefixMode()
+ );
+
+ azcopyHelper.createFolderUsingAzcopy(fs.makeQualified(testPath).toUri().getPath().substring(1));
+ // Assert that the folder is implicit
+ BlobDirectoryStateHelper.isExplicitDirectory(testPath, fs, getTestTracingContext(fs, true));
+ testGetSetXAttrHelper(fs, testPath, testPath);
+
+ // Assert that the folder is now explicit
+ BlobDirectoryStateHelper.isExplicitDirectory(testPath, fs, getTestTracingContext(fs, true));
+ }
+
+ /**
+ * Test that setting metadata over marker blob do not override
+ * x-ms-meta-hdi_IsFolder
+ * @throws Exception
+ */
+ @Test
+ public void testSetXAttrOverMarkerBlob() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ final Path testPath = new Path("ab");
+ fs.mkdirs(testPath);
+ testGetSetXAttrHelper(fs, testPath, testPath);
+
+ // Assert that the folder is now explicit
+ BlobDirectoryStateHelper.isExplicitDirectory(testPath, fs, getTestTracingContext(fs, true));
+ }
+
+ private void testGetSetXAttrHelper(final AzureBlobFileSystem fs,
+ final Path filePath, final Path testPath) throws Exception {
+
+ String attributeName1 = "user.attribute1";
+ String attributeName2 = "user.attribute2";
+ String decodedAttributeValue1;
+ String decodedAttributeValue2;
+ byte[] attributeValue1;
+ byte[] attributeValue2;
+
+ if(fs.getAbfsStore().getPrefixMode() == PrefixMode.BLOB) {
+ Assume.assumeTrue(!getIsNamespaceEnabled(fs)); // Blob endpoint Currently Supports FNS only
+ decodedAttributeValue1 = "hi";
+ decodedAttributeValue2 = "hello"; //Блюз //ä½ å¥½
+ // TODO: Modify them to unicode characters when support is added
+ attributeValue1 = fs.getAbfsStore().encodeAttribute(decodedAttributeValue1);
+ attributeValue2 = fs.getAbfsStore().encodeAttribute(decodedAttributeValue2);
+ }
+ else {
+ decodedAttributeValue1 = "hi";
+ decodedAttributeValue2 = "hello"; // DFS Endpoint only Supports ASCII
+ attributeValue1 = fs.getAbfsStore().encodeAttribute(decodedAttributeValue1);
+ attributeValue2 = fs.getAbfsStore().encodeAttribute(decodedAttributeValue2);
+ }
+
+ // Attribute not present initially
+ assertNull(fs.getXAttr(testPath, attributeName1));
+ assertNull(fs.getXAttr(testPath, attributeName2));
+
+ // Set the Attributes
+ fs.setXAttr(testPath, attributeName1, attributeValue1);
+
+ // Check if the attribute is retrievable
+ byte[] rv = fs.getXAttr(testPath, attributeName1);
+ assertTrue(Arrays.equals(rv, attributeValue1));
+ assertEquals(new String(rv, StandardCharsets.UTF_8), decodedAttributeValue1);
+
+ // Set the second Attribute
+ fs.setXAttr(testPath, attributeName2, attributeValue2);
+
+ // Check all the attributes present and previous Attribute not overridden
+ rv = fs.getXAttr(testPath, attributeName1);
+ assertTrue(Arrays.equals(rv, attributeValue1));
+ assertEquals(new String(rv, StandardCharsets.UTF_8), decodedAttributeValue1);
+ rv = fs.getXAttr(testPath, attributeName2);
+ assertTrue(Arrays.equals(rv, attributeValue2));
+ assertEquals(new String(rv, StandardCharsets.UTF_8), decodedAttributeValue2);
+ }
+
+ /**
+ * Trying to set same attribute multiple times should result in no failure
+ * @throws Exception
+ */
+ @Test
+ public void testSetXAttrMultipleOperations() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ final Path path = new Path("a/b");
+ fs.create(path);
+
+ String attributeName1 = "user.attribute1";
+ byte[] attributeValue1;
+ String decodedAttributeValue1 = "hi";
+
+ if(fs.getAbfsStore().getPrefixMode() == PrefixMode.BLOB) {
+ Assume.assumeTrue(!getIsNamespaceEnabled(fs));
+ // TODO: Modify them to unicode characters when support is added
+ attributeValue1 = fs.getAbfsStore().encodeAttribute("hi");
+ }
+ else {
+ attributeValue1 = fs.getAbfsStore().encodeAttribute("hi");
+ }
+
+ // Attribute not present initially
+ assertNull(fs.getXAttr(path, attributeName1));
+
+ // Set the Attributes Multiple times
+ // Filesystem internally adds create and replace flags
+ fs.setXAttr(path, attributeName1, attributeValue1);
+ fs.setXAttr(path, attributeName1, attributeValue1);
+
+ // Check if the attribute is retrievable
+ byte[] rv = fs.getXAttr(path, attributeName1);
+ assertTrue(Arrays.equals(rv, attributeValue1));
+ assertEquals(new String(rv, StandardCharsets.UTF_8), decodedAttributeValue1);
}
@Test
public void testSetGetXAttrCreateReplace() throws Exception {
AzureBlobFileSystem fs = getFileSystem();
- Assume.assumeTrue(getIsNamespaceEnabled(fs));
- byte[] attributeValue = fs.getAbfsStore().encodeAttribute("one");
- String attributeName = "user.someAttribute";
- Path testFile = path("createReplaceXAttr");
+ final Path testFile = new Path("a/b");
+
+ String attributeName = "user.attribute1";
+ String decodedAttributeValue1;
+ byte[] attributeValue;
+
+ if(fs.getAbfsStore().getPrefixMode() == PrefixMode.BLOB) {
+ Assume.assumeTrue(!getIsNamespaceEnabled(fs)); // Blob endpoint Currently Supports FNS only
+ decodedAttributeValue1 = "hi";
+ attributeValue = decodedAttributeValue1.getBytes(StandardCharsets.UTF_8);
+ }
+ else {
+ decodedAttributeValue1 = "hi";
+ attributeValue = fs.getAbfsStore().encodeAttribute(decodedAttributeValue1);
+ }
// after creating a file, it must be possible to create a new xAttr
- touch(testFile);
+ fs.create(testFile);
fs.setXAttr(testFile, attributeName, attributeValue, CREATE_FLAG);
assertArrayEquals(attributeValue, fs.getXAttr(testFile, attributeName));
- // however after the xAttr is created, creating it again must fail
+ // however, after the xAttr is created, creating it again must fail
intercept(IOException.class, () -> fs.setXAttr(testFile, attributeName, attributeValue, CREATE_FLAG));
}
@Test
public void testSetGetXAttrReplace() throws Exception {
AzureBlobFileSystem fs = getFileSystem();
- Assume.assumeTrue(getIsNamespaceEnabled(fs));
- byte[] attributeValue1 = fs.getAbfsStore().encodeAttribute("one");
- byte[] attributeValue2 = fs.getAbfsStore().encodeAttribute("two");
- String attributeName = "user.someAttribute";
- Path testFile = path("replaceXAttr");
+ final Path testFile = new Path("a/b");
+
+ String attributeName = "user.attribute1";
+ String decodedAttributeValue1 = "one";
+ String decodedAttributeValue2 = "two";
+
+ byte[] attributeValue1;
+ byte[] attributeValue2;
+
+ if(fs.getAbfsStore().getPrefixMode() == PrefixMode.BLOB) {
+ Assume.assumeTrue(!getIsNamespaceEnabled(fs)); // Blob endpoint Currently Supports FNS only
+ attributeValue1 = decodedAttributeValue1.getBytes(StandardCharsets.UTF_8);
+ attributeValue2 = decodedAttributeValue2.getBytes(StandardCharsets.UTF_8);
+ }
+ else {
+ attributeValue1 = fs.getAbfsStore().encodeAttribute(decodedAttributeValue1);
+ attributeValue2 = fs.getAbfsStore().encodeAttribute(decodedAttributeValue2);
+ }
// after creating a file, it must not be possible to replace an xAttr
intercept(IOException.class, () -> {
- touch(testFile);
+ fs.create(testFile);
fs.setXAttr(testFile, attributeName, attributeValue1, REPLACE_FLAG);
});
- // however after the xAttr is created, replacing it must succeed
+ // however, after the xAttr is created, replacing it must succeed
fs.setXAttr(testFile, attributeName, attributeValue1, CREATE_FLAG);
fs.setXAttr(testFile, attributeName, attributeValue2, REPLACE_FLAG);
assertArrayEquals(attributeValue2, fs.getXAttr(testFile, attributeName));
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java
index 589c3a285f48f4..449574189ad538 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java
@@ -95,12 +95,10 @@ public void testSASTokenProviderEmptySASToken() throws Exception {
testConfig.set(ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS);
testConfig.set(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, "true");
- testFs.initialize(fs.getUri(),
- this.getConfiguration().getRawConfiguration());
- intercept(SASTokenProviderException.class,
- () -> {
- testFs.create(new org.apache.hadoop.fs.Path("/testFile"));
- });
+ intercept(SASTokenProviderException.class, () -> {
+ testFs.initialize(fs.getUri(),
+ this.getConfiguration().getRawConfiguration());
+ });
}
@Test
@@ -111,11 +109,9 @@ public void testSASTokenProviderNullSASToken() throws Exception {
Configuration testConfig = this.getConfiguration().getRawConfiguration();
testConfig.set(ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS);
- testFs.initialize(fs.getUri(), this.getConfiguration().getRawConfiguration());
- intercept(SASTokenProviderException.class,
- ()-> {
- testFs.create(new org.apache.hadoop.fs.Path("/testFile"));
- });
+ intercept(SASTokenProviderException.class, () -> {
+ testFs.initialize(fs.getUri(), this.getConfiguration().getRawConfiguration());
+ });
}
@Test
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBlobConfig.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBlobConfig.java
new file mode 100644
index 00000000000000..e88aa1c511e554
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBlobConfig.java
@@ -0,0 +1,731 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Random;
+
+import org.apache.hadoop.fs.azurebfs.services.OperativeEndpoint;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
+import org.apache.hadoop.fs.azurebfs.contracts.services.AppendRequestParameters;
+import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
+import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
+
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_BLOB_ENDPOINT;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_INGRESS_FALLBACK_TO_DFS;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_MKDIRS_FALLBACK_TO_DFS;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_REDIRECT_DELETE;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_REDIRECT_RENAME;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_FS_AZURE_ENABLE_BLOBENDPOINT;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_MB;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_DNS_PREFIX;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.WASB_DNS_PREFIX;
+
+public class ITestAzureBlobFileSystemBlobConfig
+ extends AbstractAbfsIntegrationTest {
+
+ public ITestAzureBlobFileSystemBlobConfig() throws Exception {
+ super();
+ }
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ Assume.assumeFalse(getFileSystem().getIsNamespaceEnabled(
+ Mockito.mock(TracingContext.class)));
+ }
+
+ @Test
+ public void testDfsEndpointWhenBlobEndpointConfigIsDisabled()
+ throws Exception {
+ AzureBlobFileSystem fs = createFileSystemForEndpointConfigPair(
+ FS_AZURE_ENABLE_BLOB_ENDPOINT, false, true);
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ fs.getAbfsStore().setClient(client);
+ fs.create(new Path("/tmp"));
+ verifyCreatePathExecution(client);
+ }
+
+ private void verifyCreatePathExecution(final AbfsClient client)
+ throws AzureBlobFileSystemException {
+ Mockito.verify(client, Mockito.times(1))
+ .createPath(Mockito.anyString(), Mockito.anyBoolean(),
+ Mockito.anyBoolean(), Mockito.nullable(String.class),
+ Mockito.nullable(String.class),
+ Mockito.anyBoolean(), Mockito.nullable(String.class),
+ Mockito.any(TracingContext.class));
+ }
+
+ @Test
+ public void testDfsEndpointWhenBlobEndpointConfigIsEnabled()
+ throws Exception {
+ AzureBlobFileSystem fs = createFileSystemForEndpointConfigPair(
+ FS_AZURE_ENABLE_BLOB_ENDPOINT, true, true);
+ Assume.assumeTrue(!OperativeEndpoint.isIngressEnabledOnDFS(getPrefixMode(fs), fs.getAbfsStore().getAbfsConfiguration()));
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ fs.getAbfsStore().setClient(client);
+ fs.create(new Path("/tmp"));
+ verifyCreatePathBlobExecution(client);
+ }
+
+ @Test
+ public void testBlobEndpointWhenBlobEndpointConfigIsEnabled()
+ throws Exception {
+ AzureBlobFileSystem fs = createFileSystemForEndpointConfigPair(
+ FS_AZURE_ENABLE_BLOB_ENDPOINT, true, false);
+ Assume.assumeTrue(!OperativeEndpoint.isIngressEnabledOnDFS(getPrefixMode(fs), fs.getAbfsStore().getAbfsConfiguration()));
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ fs.getAbfsStore().setClient(client);
+ fs.create(new Path("/tmp"));
+ verifyCreatePathBlobExecution(client);
+ }
+
+ @Test
+ public void testBlobEndpointWhenBlobEndpointConfigIsDisabled()
+ throws Exception {
+ AzureBlobFileSystem fs = createFileSystemForEndpointConfigPair(
+ FS_AZURE_ENABLE_BLOB_ENDPOINT, false, false);
+ Assume.assumeTrue(!OperativeEndpoint.isIngressEnabledOnDFS(getPrefixMode(fs), fs.getAbfsStore().getAbfsConfiguration()));
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ fs.getAbfsStore().setClient(client);
+ fs.create(new Path("/tmp"));
+ verifyCreatePathBlobExecution(client);
+ }
+
+ @Test
+ public void testBlobEndpointWhenBlobEndpointConfigIsNull()
+ throws Exception {
+ AzureBlobFileSystem fs = createFileSystemForEndpointConfigPair(
+ FS_AZURE_ENABLE_BLOB_ENDPOINT, null, false);
+ Assume.assumeTrue(!OperativeEndpoint.isIngressEnabledOnDFS(getPrefixMode(fs), fs.getAbfsStore().getAbfsConfiguration()));
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ fs.getAbfsStore().setClient(client);
+ fs.create(new Path("/tmp"));
+ verifyCreatePathBlobExecution(client);
+ }
+
+ @Test
+ public void testBlobEndpointWithMkdirsOnDFS() throws Exception {
+ AzureBlobFileSystem fs = createFileSystemForEndpointConfigPair(
+ FS_AZURE_MKDIRS_FALLBACK_TO_DFS, true, false);
+ Assume.assumeTrue(!OperativeEndpoint.isIngressEnabledOnDFS(getPrefixMode(fs), fs.getAbfsStore().getAbfsConfiguration()));
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ fs.getAbfsStore().setClient(client);
+ int[] dirCreatedOverDFSExecCount = new int[1];
+ dirCreatedOverDFSExecCount[0] = 0;
+ int[] fileCreatedOverDFSExecCount = new int[1];
+ fileCreatedOverDFSExecCount[0] = 0;
+ checkDirAndFileCreationOnDFS(client, dirCreatedOverDFSExecCount,
+ fileCreatedOverDFSExecCount);
+ fs.mkdirs(new Path("/tmp"));
+ fs.create(new Path("/file"));
+ verifyCreatePathBlobExecution(client);
+ Assert.assertTrue(dirCreatedOverDFSExecCount[0] == 1);
+ }
+
+ @Test
+ public void testBlobEndpointWithMkdirsOnDfsNoOverride() throws Exception {
+ AzureBlobFileSystem fs = createFileSystemForEndpointConfigPair(
+ FS_AZURE_MKDIRS_FALLBACK_TO_DFS, false, false);
+ Assume.assumeTrue(!OperativeEndpoint.isIngressEnabledOnDFS(getPrefixMode(fs), fs.getAbfsStore().getAbfsConfiguration()));
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ fs.getAbfsStore().setClient(client);
+ int[] dirCreatedOverBlobExecCount = new int[1];
+ dirCreatedOverBlobExecCount[0] = 0;
+ int[] fileCreatedOverBlobExecCount = new int[1];
+ fileCreatedOverBlobExecCount[0] = 0;
+
+ checkDirAndFileCreationOnBlob(client, dirCreatedOverBlobExecCount,
+ fileCreatedOverBlobExecCount);
+
+ fs.mkdirs(new Path("/tmp"));
+ fs.create(new Path("/file"));
+ Assert.assertTrue(dirCreatedOverBlobExecCount[0] == 1);
+ Assert.assertTrue(fileCreatedOverBlobExecCount[0] == 1);
+ }
+
+ private void checkDirAndFileCreationOnBlob(final AbfsClient client,
+ final int[] dirCreatedOverBlobExecCount,
+ final int[] fileCreatedOverBlobExecCount)
+ throws AzureBlobFileSystemException {
+ Mockito.doAnswer(answer -> {
+ if (!(Boolean) answer.getArgument(1)) {
+ dirCreatedOverBlobExecCount[0]++;
+ } else {
+ fileCreatedOverBlobExecCount[0]++;
+ }
+ return answer.callRealMethod();
+ }).when(client)
+ .createPathBlob(Mockito.anyString(), Mockito.anyBoolean(),
+ Mockito.anyBoolean(), Mockito.nullable(
+ HashMap.class), Mockito.nullable(String.class),
+ Mockito.any(TracingContext.class));
+ }
+
+ @Test
+ public void testDFSEndpointWithMkdirsOnDFS() throws Exception {
+ AzureBlobFileSystem fs = createFileSystemForEndpointConfigPair(
+ FS_AZURE_MKDIRS_FALLBACK_TO_DFS, true, true);
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ fs.getAbfsStore().setClient(client);
+ int[] dirCreatedOverDFSExecCount = new int[1];
+ dirCreatedOverDFSExecCount[0] = 0;
+ int[] fileCreatedOverDFSExecCount = new int[1];
+ fileCreatedOverDFSExecCount[0] = 0;
+ checkDirAndFileCreationOnDFS(client, dirCreatedOverDFSExecCount,
+ fileCreatedOverDFSExecCount);
+ fs.mkdirs(new Path("/tmp"));
+ fs.create(new Path("/file"));
+ Assert.assertTrue(dirCreatedOverDFSExecCount[0] == 1);
+ Assert.assertTrue(fileCreatedOverDFSExecCount[0] == 1);
+ }
+
+ @Test
+ public void testDFSEndpointWithMkdirsOnDFSNoOverride() throws Exception {
+ AzureBlobFileSystem fs = createFileSystemForEndpointConfigPair(
+ FS_AZURE_MKDIRS_FALLBACK_TO_DFS, false, true);
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ fs.getAbfsStore().setClient(client);
+ int[] dirCreatedOverDFSExecCount = new int[1];
+ dirCreatedOverDFSExecCount[0] = 0;
+ int[] fileCreatedOverDFSExecCount = new int[1];
+ fileCreatedOverDFSExecCount[0] = 0;
+ checkDirAndFileCreationOnDFS(client, dirCreatedOverDFSExecCount, fileCreatedOverDFSExecCount);
+ fs.mkdirs(new Path("/tmp"));
+ fs.create(new Path("/file"));
+ Assert.assertTrue(dirCreatedOverDFSExecCount[0] == 1);
+ Assert.assertTrue(fileCreatedOverDFSExecCount[0] == 1);
+ }
+
+ @Test
+ public void testBlobEndpointNoDeleteWasbRedirect() throws Exception {
+ AzureBlobFileSystem fs = Mockito.spy(createFileSystemForEndpointConfigPair(
+ FS_AZURE_REDIRECT_DELETE, false, false));
+ Assert.assertNull(fs.getNativeFs());
+ NativeAzureFileSystem nativeAzureFileSystem = null;
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ assignStoreMocksToFs(fs, nativeAzureFileSystem, store, client);
+
+ Boolean[] isDeleteOverNativeFS = new Boolean[1];
+ isDeleteOverNativeFS[0] = false;
+ Boolean[] isDeleteOverDFSEndpoint = new Boolean[1];
+ isDeleteOverDFSEndpoint[0] = false;
+ Boolean[] isDeleteOverAbfsBlobEndpoint = new Boolean[1];
+ isDeleteOverAbfsBlobEndpoint[0] = false;
+
+ countDeleteOverAbfsAndWasb(nativeAzureFileSystem, client, isDeleteOverNativeFS,
+ isDeleteOverDFSEndpoint, isDeleteOverAbfsBlobEndpoint);
+
+ fs.create(new Path("/file"));
+ fs.delete(new Path("/file"), true);
+
+ Assert.assertTrue(isDeleteOverDFSEndpoint[0] || isDeleteOverAbfsBlobEndpoint[0]);
+ Assert.assertFalse(isDeleteOverNativeFS[0]);
+ }
+
+ @Test
+ public void testBlobEndpointWithDeleteWasbRedirect() throws Exception {
+ AzureBlobFileSystem fs = Mockito.spy(createFileSystemForEndpointConfigPair(
+ FS_AZURE_REDIRECT_DELETE, true, false));
+ Assert.assertNotNull(fs.getNativeFs());
+ NativeAzureFileSystem nativeAzureFileSystem = Mockito.spy(fs.getNativeFs());
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ assignStoreMocksToFs(fs, nativeAzureFileSystem, store, client);
+
+ Boolean[] isDeleteOverNativeFS = new Boolean[1];
+ isDeleteOverNativeFS[0] = false;
+ Boolean[] isDeleteOverDFSEndpoint = new Boolean[1];
+ isDeleteOverDFSEndpoint[0] = false;
+ Boolean[] isDeleteOverAbfsBlobEndpoint = new Boolean[1];
+ isDeleteOverAbfsBlobEndpoint[0] = false;
+
+ countDeleteOverAbfsAndWasb(nativeAzureFileSystem, client, isDeleteOverNativeFS,
+ isDeleteOverDFSEndpoint, isDeleteOverAbfsBlobEndpoint);
+
+ fs.create(new Path("/file"));
+ fs.delete(new Path("/file"), true);
+
+ Assert.assertFalse(isDeleteOverDFSEndpoint[0]);
+ Assert.assertTrue(isDeleteOverNativeFS[0]);
+ }
+
+ @Test
+ public void testDfsEndpointWithDeleteWasbRedirect() throws Exception {
+ AzureBlobFileSystem fs = Mockito.spy(createFileSystemForEndpointConfigPair(
+ FS_AZURE_REDIRECT_DELETE, true, true));
+ Assert.assertNotNull(fs.getNativeFs());
+ NativeAzureFileSystem nativeAzureFileSystem = Mockito.spy(fs.getNativeFs());
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ assignStoreMocksToFs(fs, nativeAzureFileSystem, store, client);
+
+ Boolean[] isDeleteOverNativeFS = new Boolean[1];
+ isDeleteOverNativeFS[0] = false;
+ Boolean[] isDeleteOverDFSEndpoint = new Boolean[1];
+ isDeleteOverDFSEndpoint[0] = false;
+ Boolean[] isDeleteOverAbfsBlobEndpoint = new Boolean[1];
+ isDeleteOverAbfsBlobEndpoint[0] = false;
+
+ countDeleteOverAbfsAndWasb(nativeAzureFileSystem, client, isDeleteOverNativeFS,
+ isDeleteOverDFSEndpoint, isDeleteOverAbfsBlobEndpoint);
+
+ fs.create(new Path("/file"));
+ fs.delete(new Path("/file"), true);
+
+ Assert.assertFalse(isDeleteOverDFSEndpoint[0]);
+ Assert.assertTrue(isDeleteOverNativeFS[0]);
+ }
+
+ @Test
+ public void testDfsEndpointNoDeleteWasbRedirect() throws Exception {
+ AzureBlobFileSystem fs = Mockito.spy(createFileSystemForEndpointConfigPair(
+ FS_AZURE_REDIRECT_DELETE, false, true));
+ Assert.assertNull(fs.getNativeFs());
+ NativeAzureFileSystem nativeAzureFileSystem = null;
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ assignStoreMocksToFs(fs, nativeAzureFileSystem, store, client);
+
+ Boolean[] isDeleteOverNativeFS = new Boolean[1];
+ isDeleteOverNativeFS[0] = false;
+ Boolean[] isDeleteOverDFSEndpoint = new Boolean[1];
+ isDeleteOverDFSEndpoint[0] = false;
+ Boolean[] isDeleteOverAbfsBlobEndpoint = new Boolean[1];
+ isDeleteOverAbfsBlobEndpoint[0] = false;
+
+ countDeleteOverAbfsAndWasb(nativeAzureFileSystem, client, isDeleteOverNativeFS,
+ isDeleteOverDFSEndpoint, isDeleteOverAbfsBlobEndpoint);
+
+ fs.create(new Path("/file"));
+ fs.delete(new Path("/file"), true);
+
+ Assert.assertTrue(isDeleteOverDFSEndpoint[0]);
+ Assert.assertFalse(isDeleteOverNativeFS[0]);
+ }
+
+ @Test
+ public void testBlobEndpointForNoRenameRedirectToWasb() throws Exception {
+ AzureBlobFileSystem fs = Mockito.spy(createFileSystemForEndpointConfigPair(
+ FS_AZURE_REDIRECT_RENAME, false, false));
+
+ NativeAzureFileSystem nativeAzureFileSystem = null;
+ if (fs.getNativeFs() != null) {
+ nativeAzureFileSystem = Mockito.spy(fs.getNativeFs());
+ }
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ assignStoreMocksToFs(fs, nativeAzureFileSystem, store, client);
+
+ Boolean[] isRenameOverNativeFS = new Boolean[1];
+ isRenameOverNativeFS[0] = false;
+ Boolean[] isRenameOverDFSEndpoint = new Boolean[1];
+ isRenameOverDFSEndpoint[0] = false;
+ Boolean[] isRenameOverBlobEndpoint = new Boolean[1];
+ isRenameOverBlobEndpoint[0] = false;
+
+ countRenameOverDfsBlobAndWasb(nativeAzureFileSystem, client, isRenameOverNativeFS,
+ isRenameOverDFSEndpoint, isRenameOverBlobEndpoint);
+
+ fs.create(new Path("/file1"));
+ fs.rename(new Path("/file1"), new Path("/file2"));
+ Assert.assertTrue(isRenameOverBlobEndpoint[0]);
+ Assert.assertFalse(isRenameOverDFSEndpoint[0]);
+ Assert.assertFalse(isRenameOverNativeFS[0]);
+ }
+
+ @Test
+ public void testBlobEndpointForRenameRedirectToWasb() throws Exception {
+ AzureBlobFileSystem fs = Mockito.spy(createFileSystemForEndpointConfigPair(
+ FS_AZURE_REDIRECT_RENAME, true, false));
+
+ NativeAzureFileSystem nativeAzureFileSystem = Mockito.spy(fs.getNativeFs());
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ assignStoreMocksToFs(fs, nativeAzureFileSystem, store, client);
+
+ Boolean[] isRenameOverNativeFS = new Boolean[1];
+ isRenameOverNativeFS[0] = false;
+ Boolean[] isRenameOverDFSEndpoint = new Boolean[1];
+ isRenameOverDFSEndpoint[0] = false;
+ Boolean[] isRenameOverBlobEndpoint = new Boolean[1];
+ isRenameOverBlobEndpoint[0] = false;
+
+ countRenameOverDfsBlobAndWasb(nativeAzureFileSystem, client, isRenameOverNativeFS,
+ isRenameOverDFSEndpoint, isRenameOverBlobEndpoint);
+
+ fs.create(new Path("/file1"));
+ fs.rename(new Path("/file1"), new Path("/file2"));
+ Assert.assertFalse(isRenameOverBlobEndpoint[0]);
+ Assert.assertFalse(isRenameOverDFSEndpoint[0]);
+ Assert.assertTrue(isRenameOverNativeFS[0]);
+ }
+
+ @Test
+ public void testDfsEndpointForRenameRedirectToWasb() throws Exception {
+ AzureBlobFileSystem fs = Mockito.spy(createFileSystemForEndpointConfigPair(
+ FS_AZURE_REDIRECT_RENAME, true, true));
+
+ NativeAzureFileSystem nativeAzureFileSystem = Mockito.spy(fs.getNativeFs());
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ assignStoreMocksToFs(fs, nativeAzureFileSystem, store, client);
+
+ Boolean[] isRenameOverNativeFS = new Boolean[1];
+ isRenameOverNativeFS[0] = false;
+ Boolean[] isRenameOverDFSEndpoint = new Boolean[1];
+ isRenameOverDFSEndpoint[0] = false;
+ Boolean[] isRenameOverBlobEndpoint = new Boolean[1];
+ isRenameOverBlobEndpoint[0] = false;
+
+ countRenameOverDfsBlobAndWasb(nativeAzureFileSystem, client, isRenameOverNativeFS,
+ isRenameOverDFSEndpoint, isRenameOverBlobEndpoint);
+
+ fs.create(new Path("/file1"));
+ fs.rename(new Path("/file1"), new Path("/file2"));
+ Assert.assertFalse(isRenameOverBlobEndpoint[0]);
+ Assert.assertFalse(isRenameOverDFSEndpoint[0]);
+ Assert.assertTrue(isRenameOverNativeFS[0]);
+ }
+
+ @Test
+ public void testDfsEndpointForNoRenameRedirectToWasb() throws Exception {
+ AzureBlobFileSystem fs = Mockito.spy(createFileSystemForEndpointConfigPair(
+ FS_AZURE_REDIRECT_RENAME, false, true));
+
+ NativeAzureFileSystem nativeAzureFileSystem = null;
+ if (fs.getNativeFs() != null) {
+ nativeAzureFileSystem = Mockito.spy(fs.getNativeFs());
+ }
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ assignStoreMocksToFs(fs, nativeAzureFileSystem, store, client);
+
+ Boolean[] isRenameOverNativeFS = new Boolean[1];
+ isRenameOverNativeFS[0] = false;
+ Boolean[] isRenameOverDFSEndpoint = new Boolean[1];
+ isRenameOverDFSEndpoint[0] = false;
+ Boolean[] isRenameOverBlobEndpoint = new Boolean[1];
+ isRenameOverBlobEndpoint[0] = false;
+
+ countRenameOverDfsBlobAndWasb(nativeAzureFileSystem, client, isRenameOverNativeFS,
+ isRenameOverDFSEndpoint, isRenameOverBlobEndpoint);
+
+ fs.create(new Path("/file1"));
+ fs.rename(new Path("/file1"), new Path("/file2"));
+ Assert.assertFalse(isRenameOverBlobEndpoint[0]);
+ Assert.assertTrue(isRenameOverDFSEndpoint[0]);
+ Assert.assertFalse(isRenameOverNativeFS[0]);
+ }
+
+ @Test
+ public void testBlobEndpointIngressRedirectToDfs() throws Exception {
+ AzureBlobFileSystem fs = Mockito.spy(createFileSystemForEndpointConfigPair(
+ FS_AZURE_INGRESS_FALLBACK_TO_DFS, true, false));
+
+ NativeAzureFileSystem nativeAzureFileSystem = null;
+ if (fs.getNativeFs() != null) {
+ nativeAzureFileSystem = Mockito.spy(fs.getNativeFs());
+ }
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ assignStoreMocksToFs(fs, nativeAzureFileSystem, store, client);
+
+ int[] createOverBlob = new int[1];
+ createOverBlob[0] = 0;
+ int[] createOverDfs = new int[1];
+ createOverDfs[0] = 0;
+ int[] appendOverBlob = new int[1];
+ appendOverBlob[0] = 0;
+ int[] appendOverDfs = new int[1];
+ appendOverDfs[0] = 0;
+ int[] flushOverBlob = new int[1];
+ flushOverBlob[0] = 0;
+ int[] flushOverDfs = new int[1];
+ flushOverDfs[0] = 0;
+
+ countIngressOverDfsAndBlob(client, createOverBlob, createOverDfs, appendOverBlob,
+ appendOverDfs,
+ flushOverBlob, flushOverDfs);
+
+ FSDataOutputStream outputStream = fs.create(new Path("/file"));
+ byte[] bytes = new byte[ONE_MB * 4];
+ new Random().nextBytes(bytes);
+ outputStream.write(bytes);
+ outputStream.close();
+
+ Assert.assertTrue(createOverDfs[0] == 1);
+ Assert.assertTrue(appendOverDfs[0] == 1);
+ Assert.assertTrue(flushOverDfs[0] == 1);
+ }
+
+ @Test
+ public void testBlobEndpointNoIngressRedirectToDfs() throws Exception {
+ AzureBlobFileSystem fs = Mockito.spy(createFileSystemForEndpointConfigPair(
+ FS_AZURE_INGRESS_FALLBACK_TO_DFS, false, false));
+
+ NativeAzureFileSystem nativeAzureFileSystem = null;
+ if (fs.getNativeFs() != null) {
+ nativeAzureFileSystem = Mockito.spy(fs.getNativeFs());
+ }
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ assignStoreMocksToFs(fs, nativeAzureFileSystem, store, client);
+
+ int[] createOverBlob = new int[1];
+ createOverBlob[0] = 0;
+ int[] createOverDfs = new int[1];
+ createOverDfs[0] = 0;
+ int[] appendOverBlob = new int[1];
+ appendOverBlob[0] = 0;
+ int[] appendOverDfs = new int[1];
+ appendOverDfs[0] = 0;
+ int[] flushOverBlob = new int[1];
+ flushOverBlob[0] = 0;
+ int[] flushOverDfs = new int[1];
+ flushOverDfs[0] = 0;
+
+ countIngressOverDfsAndBlob(client, createOverBlob, createOverDfs, appendOverBlob,
+ appendOverDfs,
+ flushOverBlob, flushOverDfs);
+
+ FSDataOutputStream outputStream = fs.create(new Path("/file"));
+ byte[] bytes = new byte[ONE_MB * 4];
+ new Random().nextBytes(bytes);
+ outputStream.write(bytes);
+ outputStream.close();
+
+ Assert.assertTrue(createOverBlob[0] == 1);
+ Assert.assertTrue(appendOverBlob[0] == 1);
+ Assert.assertTrue(flushOverBlob[0] == 1);
+ }
+
+ @Test
+ public void testDfsEndpointNoIngressRedirectToDfs() throws Exception {
+ AzureBlobFileSystem fs = Mockito.spy(createFileSystemForEndpointConfigPair(
+ FS_AZURE_INGRESS_FALLBACK_TO_DFS, true, true));
+
+ NativeAzureFileSystem nativeAzureFileSystem = null;
+ if (fs.getNativeFs() != null) {
+ nativeAzureFileSystem = Mockito.spy(fs.getNativeFs());
+ }
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ assignStoreMocksToFs(fs, nativeAzureFileSystem, store, client);
+
+ int[] createOverBlob = new int[1];
+ createOverBlob[0] = 0;
+ int[] createOverDfs = new int[1];
+ createOverDfs[0] = 0;
+ int[] appendOverBlob = new int[1];
+ appendOverBlob[0] = 0;
+ int[] appendOverDfs = new int[1];
+ appendOverDfs[0] = 0;
+ int[] flushOverBlob = new int[1];
+ flushOverBlob[0] = 0;
+ int[] flushOverDfs = new int[1];
+ flushOverDfs[0] = 0;
+
+ countIngressOverDfsAndBlob(client, createOverBlob, createOverDfs, appendOverBlob,
+ appendOverDfs,
+ flushOverBlob, flushOverDfs);
+
+ FSDataOutputStream outputStream = fs.create(new Path("/file"));
+ byte[] bytes = new byte[ONE_MB * 4];
+ new Random().nextBytes(bytes);
+ outputStream.write(bytes);
+ outputStream.close();
+
+ Assert.assertTrue(createOverDfs[0] == 1);
+ Assert.assertTrue(appendOverDfs[0] == 1);
+ Assert.assertTrue(flushOverDfs[0] == 1);
+ }
+
+ private void countIngressOverDfsAndBlob(final AbfsClient client,
+ final int[] createOverBlob,
+ final int[] createOverDfs,
+ final int[] appendOverBlob,
+ final int[] appendOverDfs,
+ final int[] flushOverBlob,
+ final int[] flushOverDfs) throws IOException {
+ Mockito.doAnswer(answer -> {
+ createOverDfs[0]++;
+ return answer.callRealMethod();
+ }).when(client).createPath(Mockito.anyString(), Mockito.anyBoolean(),
+ Mockito.anyBoolean(), Mockito.nullable(String.class), Mockito.nullable(String.class),
+ Mockito.anyBoolean(), Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+ Mockito.doAnswer(answer -> {
+ createOverBlob[0]++;
+ return answer.callRealMethod();
+ }).when(client).createPathBlob(Mockito.anyString(), Mockito.anyBoolean(),
+ Mockito.anyBoolean(), Mockito.nullable(HashMap.class), Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+
+ Mockito.doAnswer(answer -> {
+ appendOverDfs[0]++;
+ return answer.callRealMethod();
+ }).when(client).append(Mockito.anyString(), Mockito.any(byte[].class), Mockito.any(
+ AppendRequestParameters.class), Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+ Mockito.doAnswer(answer -> {
+ appendOverBlob[0]++;
+ return answer.callRealMethod();
+ }).when(client).append(Mockito.anyString(), Mockito.anyString(), Mockito.any(byte[].class), Mockito.any(
+ AppendRequestParameters.class), Mockito.nullable(String.class), Mockito.any(TracingContext.class), Mockito.nullable(String.class));
+
+ Mockito.doAnswer(answer -> {
+ flushOverDfs[0]++;
+ return answer.callRealMethod();
+ }).when(client).flush(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean(),
+ Mockito.anyBoolean(), Mockito.nullable(String.class),
+ Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+ Mockito.doAnswer(answer -> {
+ flushOverBlob[0]++;
+ return answer.callRealMethod();
+ }).when(client).flush(Mockito.any(byte[].class), Mockito.anyString(),
+ Mockito.anyBoolean(), Mockito.nullable(String.class),Mockito.nullable(String.class),Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+ }
+
+ private void countRenameOverDfsBlobAndWasb(final NativeAzureFileSystem nativeAzureFileSystem,
+ final AbfsClient client,
+ final Boolean[] isRenameOverNativeFS,
+ final Boolean[] isRenameOverDFSEndpoint,
+ final Boolean[] isRenameOverBlobEndpoint) throws IOException {
+ if(nativeAzureFileSystem != null) {
+ Mockito.doAnswer(answer -> {
+ isRenameOverNativeFS[0] = true;
+ return answer.callRealMethod();
+ }).when(nativeAzureFileSystem).rename(Mockito.any(Path.class), Mockito.any(Path.class));
+ }
+ Mockito.doAnswer(answer -> {
+ isRenameOverDFSEndpoint[0] = true;
+ return answer.callRealMethod();
+ }).when(client).renamePath(Mockito.anyString(), Mockito.anyString(),
+ Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+ Mockito.doAnswer(answer -> {
+ isRenameOverBlobEndpoint[0] = true;
+ return answer.callRealMethod();
+ }).when(client).copyBlob(Mockito.any(Path.class), Mockito.any(Path.class),
+ Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+ }
+
+ private void countDeleteOverAbfsAndWasb(final NativeAzureFileSystem nativeAzureFileSystem,
+ final AbfsClient client,
+ final Boolean[] isDeleteOverNativeFS,
+ final Boolean[] isDeleteOverDFSEndpoint,
+ final Boolean[] isDeleteOverAbfsBlobEndpoint) throws IOException {
+ Mockito.doAnswer(answer -> {
+ isDeleteOverDFSEndpoint[0] = true;
+ return answer.callRealMethod();
+ }).when(client).deletePath(Mockito.any(String.class), Mockito.anyBoolean(), Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+
+ Mockito.doAnswer(answer -> {
+ isDeleteOverAbfsBlobEndpoint[0] = true;
+ return answer.callRealMethod();
+ }).when(client).deleteBlobPath(Mockito.any(Path.class), Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+
+ if(nativeAzureFileSystem != null) {
+ Mockito.doAnswer(answer -> {
+ isDeleteOverNativeFS[0] = true;
+ return answer.callRealMethod();
+ }).when(nativeAzureFileSystem).delete(Mockito.any(Path.class), Mockito.anyBoolean());
+ }
+ }
+
+ private void assignStoreMocksToFs(final AzureBlobFileSystem fs,
+ final NativeAzureFileSystem nativeAzureFileSystem,
+ final AzureBlobFileSystemStore store,
+ final AbfsClient client) {
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ store.setClient(client);
+ Mockito.doReturn(nativeAzureFileSystem).when(fs).getNativeFs();
+ }
+
+ private void checkDirAndFileCreationOnDFS(final AbfsClient client,
+ final int[] dirCreatedOverDFSExecCount,
+ final int[] fileCreatedOverDFSExecCount)
+ throws AzureBlobFileSystemException {
+ Mockito.doAnswer(answer -> {
+ if (!(Boolean) answer.getArgument(1)) {
+ dirCreatedOverDFSExecCount[0]++;
+ } else {
+ fileCreatedOverDFSExecCount[0]++;
+ }
+ return answer.callRealMethod();
+ })
+ .when(client)
+ .createPath(Mockito.anyString(), Mockito.anyBoolean(),
+ Mockito.anyBoolean(), Mockito.nullable(String.class),
+ Mockito.nullable(String.class), Mockito.anyBoolean(),
+ Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+ }
+
+
+ private void verifyCreatePathBlobExecution(final AbfsClient client)
+ throws AzureBlobFileSystemException {
+ Mockito.verify(client, Mockito.times(1))
+ .createPathBlob(Mockito.anyString(), Mockito.anyBoolean(),
+ Mockito.anyBoolean(), Mockito.nullable(
+ HashMap.class), Mockito.nullable(String.class),
+ Mockito.any(TracingContext.class));
+ }
+
+ private AzureBlobFileSystem createFileSystemForEndpointConfigPair(String configName,
+ Boolean configVal,
+ Boolean dfsEndpoint) throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ Assume.assumeFalse(
+ fs.getIsNamespaceEnabled(getTestTracingContext(fs, true)));
+ Configuration configuration = Mockito.spy(getRawConfiguration());
+ if(!FS_AZURE_ENABLE_BLOB_ENDPOINT.equalsIgnoreCase(configName)) {
+ configuration.set(FS_AZURE_ENABLE_BLOB_ENDPOINT, Boolean.toString(DEFAULT_FS_AZURE_ENABLE_BLOBENDPOINT));
+ }
+ fixEndpointAsPerTest(configuration, dfsEndpoint);
+ if (configVal != null) {
+ configuration.set(configName, configVal.toString());
+ }
+ return (AzureBlobFileSystem) FileSystem.newInstance(configuration);
+ }
+
+ private void fixEndpointAsPerTest(Configuration configuration,
+ final Boolean dfsEndpoint) {
+ if (dfsEndpoint) {
+ String url = getTestUrl();
+ url = url.replace(WASB_DNS_PREFIX, ABFS_DNS_PREFIX);
+ configuration.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
+ url);
+ } else {
+ String url = getTestUrl();
+ url = url.replace(ABFS_DNS_PREFIX, WASB_DNS_PREFIX);
+ configuration.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
+ url);
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java
index 7ec03552833f09..80f26c8d46291f 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java
@@ -20,8 +20,10 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.reflect.Field;
+import java.nio.file.AccessDeniedException;
import java.util.List;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
import org.junit.Assume;
import org.junit.Test;
@@ -41,9 +43,12 @@
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_HNS_ENABLED;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_CHECK_ACCESS;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_DNS_PREFIX;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.WASB_DNS_PREFIX;
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID;
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET;
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CHECKACCESS_TEST_USER_GUID;
@@ -84,29 +89,64 @@ private void setTestUserFs() throws Exception {
if (this.testUserFs != null) {
return;
}
+ AzureBlobFileSystemStore abfsStore = getAbfsStore(getFileSystem());
+ String accountName = this.getAccountName();
+ if (abfsStore.getPrefixMode() == PrefixMode.BLOB) {
+ if (abfsStore.getAbfsConfiguration().shouldEnableBlobEndPoint()) {
+ accountName = getAccountName().replace(ABFS_DNS_PREFIX, WASB_DNS_PREFIX);
+ }
+ }
checkIfConfigIsSet(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT
- + "." + getAccountName());
+ + "." + accountName);
Configuration conf = getRawConfiguration();
- setTestFsConf(FS_AZURE_BLOB_FS_CLIENT_ID,
+ setTestFsConf(FS_AZURE_BLOB_FS_CLIENT_ID, accountName,
FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID);
- setTestFsConf(FS_AZURE_BLOB_FS_CLIENT_SECRET,
+ setTestFsConf(FS_AZURE_BLOB_FS_CLIENT_SECRET, accountName,
FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET);
conf.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.OAuth.name());
conf.set(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + "."
- + getAccountName(), ClientCredsTokenProvider.class.getName());
+ + accountName, ClientCredsTokenProvider.class.getName());
conf.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION,
false);
this.testUserFs = FileSystem.newInstance(getRawConfiguration());
}
- private void setTestFsConf(final String fsConfKey,
+ private void setTestUserFsNonHNS() throws Exception {
+ AzureBlobFileSystemStore abfsStore = getAbfsStore(getFileSystem());
+ String accountName = this.getAccountName();
+ checkIfConfigIsSet(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT
+ + "." + accountName);
+ Configuration conf = Mockito.spy(getRawConfiguration());
+ setTestFsConf1(FS_AZURE_BLOB_FS_CLIENT_ID, accountName,
+ FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID, conf);
+ setTestFsConf1(FS_AZURE_BLOB_FS_CLIENT_SECRET, accountName,
+ FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET, conf);
+ conf.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.OAuth.name());
+ conf.set(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + "."
+ + accountName, ClientCredsTokenProvider.class.getName());
+ conf.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION,
+ false);
+ conf.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, "");
+ FileSystem testUserFsNonHns;
+ testUserFsNonHns = FileSystem.newInstance(conf);
+ }
+
+ private void setTestFsConf(final String fsConfKey, String accountName,
final String testFsConfKey) {
- final String confKeyWithAccountName = fsConfKey + "." + getAccountName();
+ final String confKeyWithAccountName = fsConfKey + "." + accountName;
final String confValue = getConfiguration()
.getString(testFsConfKey, "");
getRawConfiguration().set(confKeyWithAccountName, confValue);
}
+ private void setTestFsConf1(final String fsConfKey, String accountName,
+ final String testFsConfKey, Configuration conf) {
+ final String confKeyWithAccountName = fsConfKey + "." + accountName;
+ final String confValue = getConfiguration()
+ .getString(testFsConfKey, "");
+ conf.set(confKeyWithAccountName, confValue);
+ }
+
@Test(expected = IllegalArgumentException.class)
public void testCheckAccessWithNullPath() throws IOException {
superUserFs.access(null, FsAction.READ);
@@ -174,14 +214,16 @@ public void testCheckAccessForAccountWithoutNS() throws Exception {
checkIfConfigIsSet(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET);
checkIfConfigIsSet(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_USER_GUID);
- setTestUserFs();
+ Configuration configuration = Mockito.spy(getFileSystem().getConf());
+ FileSystem testUserFSWithoutNS;
+ testUserFSWithoutNS = FileSystem.newInstance(configuration);
// When the driver does not know if the account is HNS enabled or not it
// makes a server call and fails
- intercept(AccessControlException.class,
+
+ intercept(Exception.class,
"\"This request is not authorized to perform this operation using "
- + "this permission.\", 403",
- () -> testUserFs.access(new Path("/"), FsAction.READ));
+ + "this permission.\", 403", this::setTestUserFsNonHNS);
// When the driver has already determined if the account is HNS enabled
// or not, and as the account is non HNS the AzureBlobFileSystem#access
@@ -194,8 +236,8 @@ public void testCheckAccessForAccountWithoutNS() throws Exception {
Field abfsStoreField = AzureBlobFileSystem.class.getDeclaredField(
"abfsStore");
abfsStoreField.setAccessible(true);
- abfsStoreField.set(testUserFs, mockAbfsStore);
- testUserFs.access(new Path("/"), FsAction.READ);
+ abfsStoreField.set(testUserFSWithoutNS, mockAbfsStore);
+ testUserFSWithoutNS.access(new Path("/"), FsAction.READ);
superUserFs.access(new Path("/"), FsAction.READ);
}
@@ -304,9 +346,9 @@ public void testFsActionALL() throws Exception {
}
private void checkPrerequisites() throws Exception {
- setTestUserFs();
Assume.assumeTrue(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT + " is false",
- isHNSEnabled);
+ isHNSEnabled);
+ setTestUserFs();
Assume.assumeTrue(FS_AZURE_ENABLE_CHECK_ACCESS + " is false",
isCheckAccessEnabled);
checkIfConfigIsSet(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID);
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java
index 86689a1e61940d..42c54979499dec 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java
@@ -22,17 +22,36 @@
import java.io.FilterOutputStream;
import java.io.IOException;
import java.lang.reflect.Field;
+import java.util.ArrayList;
import java.util.EnumSet;
+import java.util.List;
import java.util.UUID;
-import org.junit.Test;
-
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
+
+import java.util.HashMap;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException;
+import org.apache.hadoop.fs.azurebfs.services.OperativeEndpoint;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
+import org.apache.hadoop.test.LambdaTestUtils;
+
+import org.junit.Assert;
+import org.junit.Assume;
+
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.test.GenericTestUtils;
@@ -41,11 +60,12 @@
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.ConcurrentWriteOperationDetectedException;
import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
-import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation;
import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation;
-import org.apache.hadoop.fs.azurebfs.services.TestAbfsClient;
+import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation;
+import org.apache.hadoop.fs.azurebfs.services.ITestAbfsClient;
import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderValidator;
+import org.mockito.Mockito;
import static java.net.HttpURLConnection.HTTP_CONFLICT;
import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR;
@@ -53,7 +73,14 @@
import static java.net.HttpURLConnection.HTTP_OK;
import static java.net.HttpURLConnection.HTTP_PRECON_FAILED;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY;
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.TRUE;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_BLOB_MKDIR_OVERWRITE;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_LEASE_CREATE_NON_RECURSIVE;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_MKDIRS_FALLBACK_TO_DFS;
+import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_META_HDI_ISFOLDER;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
@@ -71,9 +98,16 @@ public class ITestAzureBlobFileSystemCreate extends
private static final Path TEST_FILE_PATH = new Path("testfile");
private static final Path TEST_FOLDER_PATH = new Path("testFolder");
private static final String TEST_CHILD_FILE = "childFile";
+ private boolean useBlobEndpoint;
public ITestAzureBlobFileSystemCreate() throws Exception {
- super();
+ super.setup();
+ AzureBlobFileSystemStore abfsStore = getAbfsStore(getFileSystem());
+ PrefixMode prefixMode = abfsStore.getPrefixMode();
+ AbfsConfiguration abfsConfiguration = abfsStore.getAbfsConfiguration();
+ useBlobEndpoint = !(OperativeEndpoint.isIngressEnabledOnDFS(prefixMode, abfsConfiguration) ||
+ OperativeEndpoint.isMkdirEnabledOnDFS(abfsConfiguration) ||
+ OperativeEndpoint.isReadEnabledOnDFS(abfsConfiguration));
}
@Test
@@ -88,6 +122,799 @@ public void testEnsureFileCreatedImmediately() throws Exception {
assertIsFile(fs, TEST_FILE_PATH);
}
+ /**
+ * Creating subdirectory on existing file path should fail.
+ * @throws Exception
+ */
+ @Test
+ public void testMkdirsFailsForSubdirectoryOfExistingFile() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(new Path("a/b/c"));
+ fs.mkdirs(new Path("a/b/d"));
+ intercept(IOException.class, () -> fs.mkdirs(new Path("a/b/c/d/e")));
+
+ assertTrue(fs.exists(new Path("a/b/c")));
+ assertTrue(fs.exists(new Path("a/b/d")));
+ // Asserting directory created still exists as explicit.
+ assertTrue(
+ BlobDirectoryStateHelper.isExplicitDirectory(new Path("a/b/d"),
+ fs, getTestTracingContext(fs, true)));
+ }
+
+ /**
+ * Try creating file same as an existing directory.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateDirectoryAndFile() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a/b/c"));
+ assertTrue(fs.exists(new Path("a/b/c")));
+ intercept(IOException.class, () -> fs.create(new Path("a/b/c")));
+
+ // Asserting that directory still exists as explicit
+ assertTrue(BlobDirectoryStateHelper.isExplicitDirectory(new Path("a/b/c"),
+ fs, getTestTracingContext(fs, true)));
+ }
+
+ /**
+ * Creating same file without specifying overwrite.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateSameFile() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(new Path("a/b/c"));
+ fs.create(new Path("a/b/c"));
+ assertTrue(fs.exists(new Path("a/b/c")));
+ }
+
+ /**
+ * Creating same file with overwrite flag set to false.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateSameFileWithOverwriteFalse() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(new Path("a/b/c"));
+ assertTrue(fs.exists(new Path("a/b/c")));
+ intercept(IOException.class, () -> fs.create(new Path("a/b/c"), false));
+ }
+
+ /**
+ * Creation of already existing subpath should fail.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateSubPath() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(new Path("a/b/c"));
+ assertTrue(fs.exists(new Path("a/b/c")));
+ intercept(IOException.class, () -> fs.create(new Path("a/b")));
+ }
+
+ /**
+ * Creating path with parent explicit.
+ */
+ @Test
+ public void testCreatePathParentExplicit() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a/b/c"));
+ assertTrue(fs.exists(new Path("a/b/c")));
+ fs.create(new Path("a/b/c/d"));
+ assertTrue(fs.exists(new Path("a/b/c/d")));
+
+ // asserting that parent stays explicit
+ assertTrue(BlobDirectoryStateHelper.isExplicitDirectory(new Path("a/b/c"),
+ fs, getTestTracingContext(fs, true)));
+ }
+
+
+ /**
+ * Test create on implicit directory with explicit parent.
+ * @throws Exception
+ */
+ @Test
+ public void testParentExplicitPathImplicit() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final AzureBlobFileSystemStore store = fs.getAbfsStore();
+ AzcopyHelper azcopyHelper = new AzcopyHelper(
+ getAccountName(),
+ getFileSystemName(),
+ getRawConfiguration(),
+ store.getPrefixMode()
+ );
+ fs.mkdirs(new Path("/explicitParent"));
+ String sourcePathName = "/explicitParent/implicitDir";
+ Path sourcePath = new Path(sourcePathName);
+ azcopyHelper.createFolderUsingAzcopy(fs.makeQualified(sourcePath).toUri().getPath().substring(1));
+
+ intercept(IOException.class, () ->
+ fs.create(sourcePath, true));
+ intercept(IOException.class, () ->
+ fs.create(sourcePath, false));
+
+ assertTrue("Parent directory should be explicit.",
+ BlobDirectoryStateHelper.isExplicitDirectory(sourcePath.getParent(),
+ fs, getTestTracingContext(fs, true)));
+ assertTrue("Path should be implicit.",
+ BlobDirectoryStateHelper.isImplicitDirectory(sourcePath,
+ fs, getTestTracingContext(fs, true)));
+ }
+
+ /**
+ * Test create on implicit directory with implicit parent
+ * @throws Exception
+ */
+ @Test
+ public void testParentImplicitPathImplicit() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final AzureBlobFileSystemStore store = fs.getAbfsStore();
+ AzcopyHelper azcopyHelper = new AzcopyHelper(
+ getAccountName(),
+ getFileSystemName(),
+ getRawConfiguration(),
+ store.getPrefixMode()
+ );
+ String parentPathName = "/implicitParent";
+ Path parentPath = new Path(parentPathName);
+ String sourcePathName = "/implicitParent/implicitDir";
+ Path sourcePath = new Path(sourcePathName);
+
+ azcopyHelper.createFolderUsingAzcopy(fs.makeQualified(parentPath).toUri().getPath().substring(1));
+ azcopyHelper.createFolderUsingAzcopy(fs.makeQualified(sourcePath).toUri().getPath().substring(1));
+
+ intercept(IOException.class, () ->
+ fs.create(sourcePath, true));
+ intercept(IOException.class, () ->
+ fs.create(sourcePath, false));
+
+
+ assertTrue("Parent directory is implicit.",
+ BlobDirectoryStateHelper.isImplicitDirectory(parentPath,
+ fs, getTestTracingContext(fs, true)));
+
+ assertTrue("Path should also be implicit.",
+ BlobDirectoryStateHelper.isImplicitDirectory(sourcePath,
+ fs, getTestTracingContext(fs, true)));
+
+ }
+
+ /**
+ * Tests create file when file exists already
+ * Verifies using eTag for overwrite = true/false
+ */
+ @Test
+ public void testCreateFileExistsImplicitParent() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final AzureBlobFileSystemStore store = fs.getAbfsStore();
+ AzcopyHelper azcopyHelper = new AzcopyHelper(
+ getAccountName(),
+ getFileSystemName(),
+ getRawConfiguration(),
+ store.getPrefixMode()
+ );
+ String parentPathName = "/implicitParent";
+ Path parentPath = new Path(parentPathName);
+ azcopyHelper.createFolderUsingAzcopy(parentPathName);
+
+ String fileName = "/implicitParent/testFile";
+ Path filePath = new Path(fileName);
+ fs.create(filePath);
+ String eTag = extractFileEtag(fileName);
+
+ // testing createFile on already existing file path
+ fs.create(filePath, true);
+
+ String eTagAfterCreateOverwrite = extractFileEtag(fileName);
+
+ assertTrue("New file eTag after create overwrite should be different from old",
+ !eTag.equals(eTagAfterCreateOverwrite));
+
+ intercept(IOException.class, () ->
+ fs.create(filePath, false));
+
+ String eTagAfterCreate = extractFileEtag(fileName);
+
+ assertTrue("File eTag should not change as creation fails",
+ eTagAfterCreateOverwrite.equals(eTagAfterCreate));
+
+ assertTrue("Parent path should also change to explicit.",
+ BlobDirectoryStateHelper.isExplicitDirectory(parentPath,
+ fs, getTestTracingContext(fs, true)));
+ }
+
+ /**
+ * Tests create file when the parent is an existing file
+ * should fail but parent directory should change to explicit
+ * @throws Exception FileAlreadyExists for blob and IOException for dfs.
+ */
+ @Test
+ public void testCreateFileParentFile() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final AzureBlobFileSystemStore store = fs.getAbfsStore();
+
+ String parentName = "/testParentFile";
+ Path parent = new Path(parentName);
+ fs.create(parent);
+
+ String childName = "/testParentFile/testChildFile";
+ Path child = new Path(childName);
+ IOException e = intercept(IOException.class, () ->
+ fs.create(child, false));
+
+ assertFalse("Parent Path should be a file.",
+ fs.getAbfsStore().getBlobProperty(parent, getTestTracingContext(fs, false))
+ .getIsDirectory());
+
+ }
+
+
+ /**
+ * Creating directory on existing file path should fail.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateMkdirs() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(new Path("a/b/c"));
+ intercept(IOException.class, () -> fs.mkdirs(new Path("a/b/c/d")));
+ }
+
+ /**
+ * Test mkdirs.
+ * @throws Exception
+ */
+ @Test
+ public void testMkdirs() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a/b"));
+ fs.mkdirs(new Path("a/b/c/d"));
+ fs.mkdirs(new Path("a/b/c/e"));
+
+ assertTrue(fs.exists(new Path("a/b")));
+ assertTrue(fs.exists(new Path("a/b/c/d")));
+ assertTrue(fs.exists(new Path("a/b/c/e")));
+
+ //Asserting that directories created as explicit
+ assertTrue(BlobDirectoryStateHelper.isExplicitDirectory(new Path("a/b"),
+ fs, getTestTracingContext(fs, true)));
+ assertTrue(
+ BlobDirectoryStateHelper.isExplicitDirectory(new Path("a/b/c/d"),
+ fs, getTestTracingContext(fs, true)));
+ assertTrue(
+ BlobDirectoryStateHelper.isExplicitDirectory(new Path("a/b/c/e"),
+ fs, getTestTracingContext(fs, true)));
+ }
+
+ /**
+ * Creating subpath of directory path should fail.
+ * @throws Exception
+ */
+ @Test
+ public void testMkdirsCreateSubPath() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a/b/c"));
+ assertTrue(fs.exists(new Path("a/b/c")));
+ intercept(IOException.class, () -> fs.create(new Path("a/b")));
+
+ //Asserting that directories created as explicit
+ assertTrue(BlobDirectoryStateHelper.isExplicitDirectory(new Path("a/b/c"),
+ fs, getTestTracingContext(fs, true)));
+ }
+
+ /**
+ * Test creation of directory by level.
+ * @throws Exception
+ */
+ @Test
+ public void testMkdirsByLevel() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a"));
+ fs.mkdirs(new Path("a/b/c"));
+ fs.mkdirs(new Path("a/b/c/d/e"));
+
+ assertTrue(fs.exists(new Path("a")));
+ assertTrue(fs.exists(new Path("a/b/c")));
+ assertTrue(fs.exists(new Path("a/b/c/d/e")));
+
+ //Asserting that directories created as explicit
+ assertTrue(BlobDirectoryStateHelper.isExplicitDirectory(new Path("a/"),
+ fs, getTestTracingContext(fs, true)));
+ assertTrue(BlobDirectoryStateHelper.isExplicitDirectory(new Path("a/b/c"),
+ fs, getTestTracingContext(fs, true)));
+ assertTrue(
+ BlobDirectoryStateHelper.isExplicitDirectory(new Path("a/b/c/d/e"),
+ fs, getTestTracingContext(fs, true)));
+ }
+
+ /*
+ Delete part of a path and validate sub path exists.
+ */
+ @Test
+ public void testMkdirsWithDelete() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a/b"));
+ fs.mkdirs(new Path("a/b/c/d"));
+ fs.delete(new Path("a/b/c/d"));
+ fs.getFileStatus(new Path("a/b/c"));
+ assertTrue(fs.exists(new Path("a/b/c")));
+ }
+
+ /**
+ * Verify mkdir and rename of parent.
+ */
+ @Test
+ public void testMkdirsWithRename() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a/b/c/d"));
+ fs.create(new Path("e/file"));
+ fs.delete(new Path("a/b/c/d"));
+ assertTrue(fs.rename(new Path("e"), new Path("a/b/c/d")));
+ assertTrue(fs.exists(new Path("a/b/c/d/file")));
+ }
+
+ /**
+ * Create a file with name /dir1 and then mkdirs for /dir1/dir2 should fail.
+ * @throws Exception
+ */
+ @Test
+ public void testFileCreateMkdirsRoot() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.setWorkingDirectory(new Path("/"));
+ final Path p1 = new Path("dir1");
+ fs.create(p1);
+ intercept(IOException.class, () -> fs.mkdirs(new Path("dir1/dir2")));
+ }
+
+ /**
+ * Create a file with name /dir1 and then mkdirs for /dir1/dir2 should fail.
+ * @throws Exception
+ */
+ @Test
+ public void testFileCreateMkdirsNonRoot() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path p1 = new Path("dir1");
+ fs.create(p1);
+ intercept(IOException.class, () -> fs.mkdirs(new Path("dir1/dir2")));
+ }
+
+ /**
+ * Creation of same directory without overwrite flag should pass.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateSameDirectory() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a/b/c"));
+ fs.mkdirs(new Path("a/b/c"));
+
+ assertTrue(fs.exists(new Path("a/b/c")));
+ //Asserting that directories created as explicit
+ assertTrue(BlobDirectoryStateHelper.isExplicitDirectory(new Path("a/b/c"),
+ fs, getTestTracingContext(fs, true)));
+ }
+
+ /**
+ * Creation of same directory without overwrite flag should pass.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateSamePathDirectory() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(new Path("a"));
+ intercept(IOException.class, () -> fs.mkdirs(new Path("a")));
+ }
+
+ /**
+ * Creation of directory with root as parent
+ */
+ @Test
+ public void testMkdirOnRootAsParent() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path path = new Path("a");
+ fs.setWorkingDirectory(new Path("/"));
+ fs.mkdirs(path);
+
+ // Asserting that the directory created by mkdir exists as explicit.
+ assertTrue(BlobDirectoryStateHelper.isExplicitDirectory(path,
+ fs, getTestTracingContext(fs, true)));
+ }
+
+ /**
+ * Creation of directory on root
+ */
+ @Test
+ public void testMkdirOnRoot() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path path = new Path("/");
+ fs.setWorkingDirectory(new Path("/"));
+ fs.mkdirs(path);
+
+ assertTrue(BlobDirectoryStateHelper.isExplicitDirectory(path,
+ fs, getTestTracingContext(fs, true)));
+ }
+
+ /**
+ * Creation of directory on path with unicode chars
+ */
+ @Test
+ public void testMkdirUnicode() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path path = new Path("/dir\u0031");
+ fs.mkdirs(path);
+
+ // Asserting that the directory created by mkdir exists as explicit.
+ assertTrue(BlobDirectoryStateHelper.isExplicitDirectory(path,
+ fs, getTestTracingContext(fs, true)));
+ }
+
+ /**
+ * Creation of directory on same path with parallel threads
+ */
+ @Test
+ public void testMkdirParallelRequests() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path path = new Path("/dir1");
+
+ ExecutorService es = Executors.newFixedThreadPool(3);
+
+ List> tasks = new ArrayList<>();
+
+ for (int i = 0; i < 3; i++) {
+ CompletableFuture future = CompletableFuture.runAsync(() -> {
+ try {
+ fs.mkdirs(path);
+ } catch (IOException e) {
+ throw new CompletionException(e);
+ }
+ }, es);
+ tasks.add(future);
+ }
+
+ // Wait for all the tasks to complete
+ CompletableFuture.allOf(tasks.toArray(new CompletableFuture[0])).join();
+
+ // Assert that the directory created by mkdir exists as explicit
+ assertTrue(BlobDirectoryStateHelper.isExplicitDirectory(path,
+ fs, getTestTracingContext(fs, true)));
+
+ }
+
+
+ /**
+ * Creation of directory with overwrite set to false should not fail according to DFS code.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateSameDirectoryOverwriteFalse() throws Exception {
+ Configuration configuration = getRawConfiguration();
+ configuration.setBoolean(FS_AZURE_ENABLE_BLOB_MKDIR_OVERWRITE, false);
+ AzureBlobFileSystem fs1 = (AzureBlobFileSystem) FileSystem.newInstance(configuration);
+ fs1.mkdirs(new Path("a/b/c"));
+ fs1.mkdirs(new Path("a/b/c"));
+
+ //Asserting that directories created as explicit
+ assertTrue(BlobDirectoryStateHelper.isExplicitDirectory(new Path("a/b/c"),
+ fs1, getTestTracingContext(fs1, true)));
+ }
+
+ /**
+ * Try creating directory same as an existing file.
+ */
+ @Test
+ public void testCreateDirectoryAndFileRecreation() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a/b/c"));
+ fs.create(new Path("a/b/c/d"));
+ assertTrue(fs.exists(new Path("a/b/c")));
+ assertTrue(fs.exists(new Path("a/b/c/d")));
+ intercept(IOException.class, () -> fs.mkdirs(new Path("a/b/c/d")));
+ }
+
+ /**
+ * Creation of directory with parent directory existing as implicit.
+ * And the directory to be created does not exist
+ * @throws Exception
+ */
+ @Test
+ public void testMkdirOnNonExistingPathWithImplicitParentDir() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path implicitPath = new Path("dir1");
+ final Path path = new Path("dir1/dir2");
+
+ // Creating implicit directory to be used as parent
+ AzcopyHelper azcopyHelper = new AzcopyHelper(
+ getAccountName(),
+ getFileSystemName(),
+ getFileSystem().getAbfsStore().getAbfsConfiguration().getRawConfiguration(),
+ fs.getAbfsStore().getPrefixMode());
+ azcopyHelper.createFolderUsingAzcopy(
+ getFileSystem().makeQualified(implicitPath).toUri().getPath().substring(1));
+
+ // Creating a directory on non-existing path inside an implicit directory
+ fs.mkdirs(path);
+
+ // Asserting that path created by azcopy becomes explicit.
+ assertTrue(BlobDirectoryStateHelper.isExplicitDirectory(implicitPath,
+ fs, getTestTracingContext(fs, true)));
+
+ // Asserting that the directory created by mkdir exists as explicit.
+ assertTrue(BlobDirectoryStateHelper.isExplicitDirectory(path,
+ fs, getTestTracingContext(fs, true)));
+ }
+
+ /**
+ * Creation of directory with parent directory existing as implicit.
+ * And the directory to be created existing as explicit directory
+ * @throws Exception
+ */
+ @Test
+ public void testMkdirOnExistingExplicitDirWithImplicitParentDir() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path implicitPath = new Path("dir1");
+ final Path path = new Path("dir1/dir2");
+
+ // Creating implicit directory to be used as parent
+ AzcopyHelper azcopyHelper = new AzcopyHelper(
+ getAccountName(),
+ getFileSystemName(),
+ getFileSystem().getAbfsStore().getAbfsConfiguration().getRawConfiguration(),
+ fs.getAbfsStore().getPrefixMode());
+ azcopyHelper.createFolderUsingAzcopy(
+ getFileSystem().makeQualified(implicitPath).toUri().getPath().substring(1));
+
+ // Creating an explicit directory on the path first
+ fs.mkdirs(path);
+
+ // Creating a directory on existing explicit directory inside an implicit directory
+ fs.mkdirs(path);
+
+ // Asserting that path created by azcopy becomes explicit.
+ assertTrue(BlobDirectoryStateHelper.isExplicitDirectory(implicitPath,
+ fs, getTestTracingContext(fs, true)));
+
+ // Asserting that the directory created by mkdir exists as explicit.
+ assertTrue(BlobDirectoryStateHelper.isExplicitDirectory(path,
+ fs, getTestTracingContext(fs, true)));
+ }
+
+ /**
+ * Creation of directory with parent directory existing as explicit.
+ * And the directory to be created existing as implicit directory
+ * @throws Exception
+ */
+ @Test
+ public void testMkdirOnExistingImplicitDirWithExplicitParentDir() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path explicitPath = new Path("dir1");
+ final Path path = new Path("dir1/dir2");
+
+ // Creating an explicit directory to be used a parent
+ fs.mkdirs(explicitPath);
+
+ // Creating implicit directory on the path
+ AzcopyHelper azcopyHelper = new AzcopyHelper(
+ getAccountName(),
+ getFileSystemName(),
+ getFileSystem().getAbfsStore().getAbfsConfiguration().getRawConfiguration(),
+ fs.getAbfsStore().getPrefixMode());
+ azcopyHelper.createFolderUsingAzcopy(
+ getFileSystem().makeQualified(path).toUri().getPath().substring(1));
+
+ // Creating a directory on existing implicit directory inside an explicit directory
+ fs.mkdirs(path);
+
+ // Asserting that the directory created by mkdir exists as explicit.
+ assertTrue(BlobDirectoryStateHelper.isExplicitDirectory(explicitPath,
+ fs, getTestTracingContext(fs, true)));
+
+ // Asserting that the directory created by mkdir exists as explicit.
+ assertTrue(BlobDirectoryStateHelper.isExplicitDirectory(path,
+ fs, getTestTracingContext(fs, true)));
+ }
+
+ /**
+ * Creation of directory with parent directory existing as implicit.
+ * And the directory to be created existing as implicit directory
+ * @throws Exception
+ */
+ @Test
+ public void testMkdirOnExistingImplicitDirWithImplicitParentDir() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path implicitPath = new Path("dir1");
+ final Path path = new Path("dir1/dir2");
+
+ // Creating implicit directory to be used as parent
+ AzcopyHelper azcopyHelper = new AzcopyHelper(
+ getAccountName(),
+ getFileSystemName(),
+ getFileSystem().getAbfsStore().getAbfsConfiguration().getRawConfiguration(),
+ fs.getAbfsStore().getPrefixMode());
+ azcopyHelper.createFolderUsingAzcopy(
+ getFileSystem().makeQualified(implicitPath).toUri().getPath().substring(1));
+
+ // Creating an implicit directory on path
+ azcopyHelper.createFolderUsingAzcopy(
+ getFileSystem().makeQualified(path).toUri().getPath().substring(1));
+
+ // Creating a directory on existing implicit directory inside an implicit directory
+ fs.mkdirs(path);
+
+ // Asserting that path created by azcopy becomes explicit.
+ assertTrue(BlobDirectoryStateHelper.isExplicitDirectory(implicitPath,
+ fs, getTestTracingContext(fs, true)));
+
+ // Asserting that the directory created by mkdir exists as explicit.
+ assertTrue(BlobDirectoryStateHelper.isExplicitDirectory(path,
+ fs, getTestTracingContext(fs, true)));
+ }
+
+ /**
+ * Creation of directory with parent directory existing as implicit.
+ * And the directory to be created existing as file
+ * @throws Exception
+ */
+ @Test
+ public void testMkdirOnExistingFileWithImplicitParentDir() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path implicitPath = new Path("dir1");
+ final Path path = new Path("dir1/dir2");
+
+ // Creating implicit directory to be used as parent
+ AzcopyHelper azcopyHelper = new AzcopyHelper(
+ getAccountName(),
+ getFileSystemName(),
+ getFileSystem().getAbfsStore().getAbfsConfiguration().getRawConfiguration(),
+ fs.getAbfsStore().getPrefixMode());
+ azcopyHelper.createFolderUsingAzcopy(
+ getFileSystem().makeQualified(implicitPath).toUri().getPath().substring(1));
+
+ // Creating a file on path
+ fs.create(path);
+
+ // Creating a directory on existing file inside an implicit directory
+ // Asserting that the mkdir fails
+ LambdaTestUtils.intercept(FileAlreadyExistsException.class, () -> {
+ fs.mkdirs(path);
+ });
+
+ // Asserting that path created by azcopy becomes explicit.
+ assertTrue(BlobDirectoryStateHelper.isExplicitDirectory(implicitPath,
+ fs, getTestTracingContext(fs, true)));
+
+ // Asserting that the file still exists at path.
+ assertFalse(
+ fs.getAbfsStore().getBlobProperty(
+ new Path(getFileSystem().makeQualified(path).toUri().getPath()),
+ getTestTracingContext(fs, true)
+ ).getIsDirectory()
+ );
+ }
+
+ /**
+ * Test to validate that if prefix mode is BLOB and client encryption key is not null, exception is thrown.
+ */
+ @Test
+ public void testCPKOverBlob() throws Exception {
+ AzureBlobFileSystemStore abfsStore = getAbfsStore(getFileSystem());
+ Assume.assumeTrue(abfsStore.getPrefixMode() == PrefixMode.BLOB);
+ Configuration configuration = Mockito.spy(getRawConfiguration());
+ String accountName = getAccountName();
+
+ configuration.set(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + accountName, "abcd");
+ intercept(InvalidConfigurationValueException.class, () -> (AzureBlobFileSystem) FileSystem.newInstance(configuration));
+ }
+
+ /**
+ * Test to validate that if prefix mode is BLOB and even if client encryption key empty, exception is thrown.
+ */
+ @Test
+ public void testCPKOverBlobEmptyKey() throws Exception {
+ AzureBlobFileSystemStore abfsStore = getAbfsStore(getFileSystem());
+ Assume.assumeTrue(abfsStore.getPrefixMode() == PrefixMode.BLOB);
+ Configuration configuration = Mockito.spy(getRawConfiguration());
+ String accountName = getAccountName();
+
+ configuration.set(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + accountName, "");
+ intercept(InvalidConfigurationValueException.class, () -> (AzureBlobFileSystem) FileSystem.newInstance(configuration));
+ }
+
+ /**
+ * Test to validate that if prefix mode is BLOB and if client encryption key is not set, no exception is thrown.
+ */
+ @Test
+ public void testCPKOverBlobNullKey() throws Exception {
+ Assume.assumeTrue(getFileSystem().getAbfsStore().getPrefixMode() == PrefixMode.BLOB);
+ Configuration configuration = Mockito.spy(getRawConfiguration());
+ AzureBlobFileSystem fs1 = (AzureBlobFileSystem) FileSystem.newInstance(configuration);
+ }
+
+ /**
+ * 1. a/b/c as implicit.
+ * 2. Create marker for b.
+ * 3. Do mkdir on a/b/c/d.
+ * 4. Verify all b,c,d have marker but a is implicit.
+ */
+ @Test
+ public void testImplicitExplicitFolder() throws Exception {
+ Configuration configuration = Mockito.spy(getRawConfiguration());
+ configuration.setBoolean(FS_AZURE_MKDIRS_FALLBACK_TO_DFS, false);
+ final AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.newInstance(configuration);
+ final Path implicitPath = new Path("a/b/c");
+
+ AzcopyHelper azcopyHelper = new AzcopyHelper(
+ getAccountName(),
+ getFileSystemName(),
+ getFileSystem().getAbfsStore().getAbfsConfiguration().getRawConfiguration(),
+ fs.getAbfsStore().getPrefixMode());
+ azcopyHelper.createFolderUsingAzcopy(
+ getFileSystem().makeQualified(implicitPath).toUri().getPath().substring(1));
+
+ Path path = makeQualified(new Path("a/b"));
+ HashMap metadata = new HashMap<>();
+ metadata.put(X_MS_META_HDI_ISFOLDER, TRUE);
+ fs.getAbfsStore().createFile(path, null, true,
+ null, null, getTestTracingContext(fs, true), metadata);
+
+ fs.mkdirs(new Path("a/b/c/d"));
+
+ assertTrue(BlobDirectoryStateHelper.isImplicitDirectory(new Path("a"),
+ fs, getTestTracingContext(fs, true)));
+ assertTrue(BlobDirectoryStateHelper.isExplicitDirectory(new Path("a/b"),
+ fs, getTestTracingContext(fs, true)));
+ assertTrue(BlobDirectoryStateHelper.isExplicitDirectory(new Path("a/b/c"),
+ fs, getTestTracingContext(fs, true)));
+ assertTrue(
+ BlobDirectoryStateHelper.isExplicitDirectory(new Path("a/b/c/d"),
+ fs, getTestTracingContext(fs, true)));
+ }
+
+ /**
+ * 1. a/b/c implicit.
+ * 2. Marker for a and c.
+ * 3. mkdir on a/b/c/d.
+ * 4. Verify a,c,d are explicit but b is implicit.
+ */
+ @Test
+ public void testImplicitExplicitFolder1() throws Exception {
+ Configuration configuration = Mockito.spy(getRawConfiguration());
+ configuration.setBoolean(FS_AZURE_MKDIRS_FALLBACK_TO_DFS, false);
+ final AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.newInstance(configuration);
+ final Path implicitPath = new Path("a/b/c");
+
+ AzcopyHelper azcopyHelper = new AzcopyHelper(
+ getAccountName(),
+ getFileSystemName(),
+ getFileSystem().getAbfsStore().getAbfsConfiguration().getRawConfiguration(),
+ fs.getAbfsStore().getPrefixMode());
+ azcopyHelper.createFolderUsingAzcopy(
+ getFileSystem().makeQualified(implicitPath).toUri().getPath().substring(1));
+
+ Path path = makeQualified(new Path("a"));
+ HashMap metadata = new HashMap<>();
+ metadata.put(X_MS_META_HDI_ISFOLDER, TRUE);
+ fs.getAbfsStore().createFile(path, null, true,
+ null, null, getTestTracingContext(fs, true), metadata);
+
+ fs.getAbfsStore().createFile(makeQualified(new Path("a/b/c")), null, true,
+ null, null, getTestTracingContext(fs, true), metadata);
+
+ fs.mkdirs(new Path("a/b/c/d"));
+
+ assertTrue(BlobDirectoryStateHelper.isImplicitDirectory(new Path("a/b"),
+ fs, getTestTracingContext(fs, true)));
+
+ // Asserting that the directory created by mkdir exists as explicit.
+ assertTrue(BlobDirectoryStateHelper.isExplicitDirectory(new Path("a"),
+ fs, getTestTracingContext(fs, true)));
+ assertTrue(BlobDirectoryStateHelper.isExplicitDirectory(new Path("a/b/c"),
+ fs, getTestTracingContext(fs, true)));
+ assertTrue(
+ BlobDirectoryStateHelper.isExplicitDirectory(new Path("a/b/c/d"),
+ fs, getTestTracingContext(fs, true)));
+ }
+
@Test
@SuppressWarnings("deprecation")
public void testCreateNonRecursive() throws Exception {
@@ -143,6 +970,151 @@ public void testCreateNonRecursive2() throws Exception {
assertIsFile(fs, testFile);
}
+ @Test
+ @SuppressWarnings("deprecation")
+ public void testCreateNonRecursiveOnRoot() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+
+ Path testFile = new Path("/");
+ try {
+ fs.createNonRecursive(testFile, FsPermission.getDefault(), false, 1024, (short) 1, 1024, null);
+ fail("Should've thrown AbfsRestOperationException with HTTP_CONFLICT");
+ } catch (AbfsRestOperationException e) {
+ assertEquals(e.getStatusCode(), HTTP_CONFLICT);
+ }
+ }
+
+ @Test
+ public void testCreateNonRecursiveForAtomicDirectoryFile() throws Exception {
+ AzureBlobFileSystem fileSystem = getFileSystem();
+ Assume.assumeTrue(
+ fileSystem.getAbfsStore().getAbfsConfiguration().getPrefixMode()
+ == PrefixMode.BLOB);
+ fileSystem.setWorkingDirectory(new Path("/"));
+ fileSystem.mkdirs(new Path("/hbase/dir"));
+ fileSystem.createFile(new Path("/hbase/dir/file"))
+ .overwrite(false)
+ .replication((short) 1)
+ .bufferSize(1024)
+ .blockSize(1024)
+ .build();
+ Assert.assertTrue(fileSystem.exists(new Path("/hbase/dir/file")));
+ }
+
+ @Test
+ public void testActiveCreateNonRecursiveDenyParallelReadOnAtomicDir() throws Exception {
+ Assume.assumeTrue(getPrefixMode(getFileSystem()) == PrefixMode.BLOB);
+ Configuration configuration = Mockito.spy(getRawConfiguration());
+ configuration.set(FS_AZURE_LEASE_CREATE_NON_RECURSIVE, "true");
+ AzureBlobFileSystem fileSystem = (AzureBlobFileSystem) FileSystem.newInstance(configuration);
+ AbfsClient client = Mockito.spy(fileSystem.getAbfsClient());
+ fileSystem.getAbfsStore().setClient(client);
+ fileSystem.setWorkingDirectory(new Path("/"));
+ fileSystem.mkdirs(new Path("/hbase/dir"));
+ fileSystem.create(new Path("/hbase/dir/file"));
+ AtomicBoolean createCalled = new AtomicBoolean(false);
+ AtomicBoolean parallelRenameDone = new AtomicBoolean(false);
+ AtomicBoolean exceptionCaught = new AtomicBoolean(false);
+
+ Mockito.doAnswer(answer -> {
+ AbfsRestOperation op = (AbfsRestOperation) answer.callRealMethod();
+ createCalled.set(true);
+ while(!parallelRenameDone.get());
+ return op;
+ }).when(client).createPathBlob(Mockito.anyString(), Mockito.anyBoolean(),
+ Mockito.anyBoolean(), Mockito.nullable(HashMap.class), Mockito.nullable(String.class), Mockito.nullable(TracingContext.class));
+
+ Mockito.doAnswer(answer -> {
+ AbfsRestOperation op = (AbfsRestOperation) answer.callRealMethod();
+ createCalled.set(true);
+ while (!parallelRenameDone.get());
+ return op;
+ })
+ .when(client)
+ .createPath(Mockito.anyString(), Mockito.anyBoolean(),
+ Mockito.anyBoolean(), Mockito.nullable(String.class),
+ Mockito.nullable(String.class), Mockito.anyBoolean(),
+ Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+
+ new Thread(() -> {
+ try {
+ while(!createCalled.get());
+ getFileSystem().rename(new Path("/hbase/dir/"), new Path("/hbase/dir2"));
+ } catch (Exception e) {
+ exceptionCaught.set(true);
+ } finally {
+ parallelRenameDone.set(true);
+ }
+ }).start();
+
+ fileSystem.createFile(new Path("/hbase/dir/file1"))
+ .overwrite(false)
+ .replication((short) 1)
+ .bufferSize(1024)
+ .blockSize(1024)
+ .build();
+
+ Assert.assertTrue(exceptionCaught.get());
+ Assert.assertTrue(fileSystem.exists(new Path("/hbase/dir/file")));
+ }
+
+ @Test
+ public void testActiveCreateNonRecursiveNotDenyParallelRenameOnAtomicDirIfLeaseConfigDisabled() throws Exception {
+ Assume.assumeTrue(getPrefixMode(getFileSystem()) == PrefixMode.BLOB);
+ Configuration configuration = Mockito.spy(getRawConfiguration());
+ AzureBlobFileSystem fileSystem = (AzureBlobFileSystem) FileSystem.newInstance(configuration);
+ AbfsClient client = Mockito.spy(fileSystem.getAbfsClient());
+ fileSystem.getAbfsStore().setClient(client);
+ fileSystem.setWorkingDirectory(new Path("/"));
+ fileSystem.mkdirs(new Path("/hbase/dir"));
+ fileSystem.create(new Path("/hbase/dir/file"));
+ AtomicBoolean createCalled = new AtomicBoolean(false);
+ AtomicBoolean parallelRenameDone = new AtomicBoolean(false);
+ AtomicBoolean exceptionCaught = new AtomicBoolean(false);
+
+ Mockito.doAnswer(answer -> {
+ AbfsRestOperation op = (AbfsRestOperation) answer.callRealMethod();
+ createCalled.set(true);
+ while(!parallelRenameDone.get());
+ return op;
+ }).when(client).createPathBlob(Mockito.anyString(), Mockito.anyBoolean(),
+ Mockito.anyBoolean(), Mockito.nullable(HashMap.class), Mockito.nullable(String.class), Mockito.nullable(TracingContext.class));
+
+ Mockito.doAnswer(answer -> {
+ AbfsRestOperation op = (AbfsRestOperation) answer.callRealMethod();
+ createCalled.set(true);
+ while (!parallelRenameDone.get());
+ return op;
+ })
+ .when(client)
+ .createPath(Mockito.anyString(), Mockito.anyBoolean(),
+ Mockito.anyBoolean(), Mockito.nullable(String.class),
+ Mockito.nullable(String.class), Mockito.anyBoolean(),
+ Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+
+ new Thread(() -> {
+ try {
+ while(!createCalled.get());
+ getFileSystem().rename(new Path("/hbase/dir/"), new Path("/hbase/dir2"));
+ } catch (Exception e) {
+ exceptionCaught.set(true);
+ } finally {
+ parallelRenameDone.set(true);
+ }
+ }).start();
+
+ fileSystem.createFile(new Path("/hbase/dir/file1"))
+ .overwrite(false)
+ .replication((short) 1)
+ .bufferSize(1024)
+ .blockSize(1024)
+ .build();
+
+ Assert.assertFalse(exceptionCaught.get());
+ Assert.assertFalse(fileSystem.exists(new Path("/hbase/dir/file")));
+ Assert.assertTrue(fileSystem.exists(new Path("/hbase/dir2/file")));
+ }
+
/**
* Attempts to use to the ABFS stream after it is closed.
*/
@@ -178,7 +1150,7 @@ public void testTryWithResources() throws Throwable {
out.write('2');
out.hsync();
fail("Expected a failure");
- } catch (FileNotFoundException fnfe) {
+ } catch (IOException fnfe) {
//appendblob outputStream does not generate suppressed exception on close as it is
//single threaded code
if (!fs.getAbfsStore().isAppendBlobKey(fs.makeQualified(testPath).toString())) {
@@ -204,7 +1176,7 @@ public void testFilterFSWriteAfterClose() throws Throwable {
final AzureBlobFileSystem fs = getFileSystem();
Path testPath = new Path(TEST_FOLDER_PATH, TEST_CHILD_FILE);
FSDataOutputStream out = fs.create(testPath);
- intercept(FileNotFoundException.class,
+ intercept(IOException.class,
() -> {
try (FilterOutputStream fos = new FilterOutputStream(out)) {
fos.write('a');
@@ -212,12 +1184,12 @@ public void testFilterFSWriteAfterClose() throws Throwable {
out.hsync();
fs.delete(testPath, false);
// trigger the first failure
- throw intercept(FileNotFoundException.class,
+ throw intercept(IOException.class,
() -> {
- fos.write('b');
- out.hsync();
- return "hsync didn't raise an IOE";
- });
+ fos.write('b');
+ out.hsync();
+ return "hsync didn't raise an IOE";
+ });
}
});
}
@@ -262,13 +1234,18 @@ public void testCreateFileOverwrite(boolean enableConditionalCreateOverwrite)
fs.create(nonOverwriteFile, false);
// One request to server to create path should be issued
- createRequestCount++;
+ // two calls added for -
+ // 1. getFileStatus on DFS endpoint : 1
+ // getFileStatus on Blob endpoint: 2 (Additional List blob call)
+ // 2. actual create call: 1
+ createRequestCount += (useBlobEndpoint ? 3: 2);
assertAbfsStatistics(
CONNECTIONS_MADE,
totalConnectionMadeBeforeTest + createRequestCount,
fs.getInstrumentationMap());
+
// Case 2: Not Overwrite - File pre-exists
fs.registerListener(new TracingHeaderValidator(
fs.getAbfsStore().getAbfsConfiguration().getClientCorrelationId(),
@@ -278,7 +1255,11 @@ public void testCreateFileOverwrite(boolean enableConditionalCreateOverwrite)
fs.registerListener(null);
// One request to server to create path should be issued
- createRequestCount++;
+ // Only single tryGetFileStatus should happen
+ // 1. getFileStatus on DFS endpoint : 1
+ // getFileStatus on Blob endpoint: 1 (No Additional List blob call as file exists)
+
+ createRequestCount ++;
assertAbfsStatistics(
CONNECTIONS_MADE,
@@ -292,8 +1273,12 @@ public void testCreateFileOverwrite(boolean enableConditionalCreateOverwrite)
// create should be successful
fs.create(overwriteFilePath, true);
- // One request to server to create path should be issued
- createRequestCount++;
+ /// One request to server to create path should be issued
+ // two calls added for -
+ // 1. getFileStatus on DFS endpoint : 1
+ // getFileStatus on Blob endpoint: 2 (Additional List blob call for non-existing path)
+ // 2. actual create call: 1
+ createRequestCount += (useBlobEndpoint ? 3: 2);
assertAbfsStatistics(
CONNECTIONS_MADE,
@@ -307,6 +1292,14 @@ public void testCreateFileOverwrite(boolean enableConditionalCreateOverwrite)
fs.create(overwriteFilePath, true);
fs.registerListener(null);
+ // One request to server to create path should be issued
+ // First tryGetFileStatus should happen
+ // 1. getFileStatus on DFS endpoint : 1
+ // getFileStatus on Blob endpoint: 1 (No Additional List blob call as file exists)
+
+ createRequestCount ++;
+
+ // Second actual create call will hap
if (enableConditionalCreateOverwrite) {
// Three requests will be sent to server to create path,
// 1. create without overwrite
@@ -350,17 +1343,18 @@ public void testNegativeScenariosForCreateOverwriteDisabled()
Boolean.toString(true));
final AzureBlobFileSystem fs =
- (AzureBlobFileSystem) FileSystem.newInstance(currentFs.getUri(),
- config);
+ Mockito.spy((AzureBlobFileSystem) FileSystem.newInstance(currentFs.getUri(),
+ config));
// Get mock AbfsClient with current config
AbfsClient
mockClient
- = TestAbfsClient.getMockAbfsClient(
+ = ITestAbfsClient.getMockAbfsClient(
fs.getAbfsStore().getClient(),
fs.getAbfsStore().getAbfsConfiguration());
- AzureBlobFileSystemStore abfsStore = fs.getAbfsStore();
+ AzureBlobFileSystemStore abfsStore = Mockito.spy(fs.getAbfsStore());
+ Mockito.doReturn(abfsStore).when(fs).getAbfsStore();
abfsStore = setAzureBlobSystemStoreField(abfsStore, "client", mockClient);
boolean isNamespaceEnabled = abfsStore
.getIsNamespaceEnabled(getTestTracingContext(fs, false));
@@ -369,6 +1363,7 @@ public void testNegativeScenariosForCreateOverwriteDisabled()
AbfsRestOperation.class);
AbfsHttpOperation http200Op = mock(
AbfsHttpOperation.class);
+ AzureBlobFileSystemStore.VersionedFileStatus fileStatus = mock(AzureBlobFileSystemStore.VersionedFileStatus.class);
when(http200Op.getStatusCode()).thenReturn(HTTP_OK);
when(successOp.getResult()).thenReturn(http200Op);
@@ -396,12 +1391,34 @@ public void testNegativeScenariosForCreateOverwriteDisabled()
isNamespaceEnabled ? any(String.class) : eq(null),
any(boolean.class), eq(null), any(TracingContext.class));
+ // mock for overwrite=false
+ doThrow(conflictResponseEx) // Scn1: GFS fails with Http404
+ .doThrow(conflictResponseEx) // Scn2: GFS fails with Http500
+ .doThrow(
+ conflictResponseEx) // Scn3: create overwrite=true fails with Http412
+ .doThrow(
+ conflictResponseEx) // Scn4: create overwrite=true fails with Http500
+ .doThrow(
+ serverErrorResponseEx) // Scn5: create overwrite=false fails with Http500
+ .when(mockClient)
+ .createPathBlob(any(String.class), eq(true), eq(false),
+ any(), eq(null), any(TracingContext.class));
+
doThrow(fileNotFoundResponseEx) // Scn1: GFS fails with Http404
- .doThrow(serverErrorResponseEx) // Scn2: GFS fails with Http500
- .doReturn(successOp) // Scn3: create overwrite=true fails with Http412
- .doReturn(successOp) // Scn4: create overwrite=true fails with Http500
+ .doThrow(serverErrorResponseEx) // Scn2: GFS fails with Http500
+ .doReturn(fileStatus)
+ .doReturn(fileStatus)
+ .when(abfsStore)
+ .getFileStatus(any(Path.class), any(TracingContext.class), anyBoolean());
+
+ // mock for overwrite=true
+ doThrow(
+ preConditionResponseEx) // Scn3: create overwrite=true fails with Http412
+ .doThrow(
+ serverErrorResponseEx) // Scn4: create overwrite=true fails with Http500
.when(mockClient)
- .getPathStatus(any(String.class), eq(false), any(TracingContext.class));
+ .createPathBlob(any(String.class), eq(true), eq(true),
+ any(), eq(null), any(TracingContext.class));
// mock for overwrite=true
doThrow(
@@ -479,10 +1496,74 @@ private void validateCreateFileException(final Class ex
intercept(
exceptionClass,
() -> abfsStore.createFile(testPath, null, true, permission, umask,
- getTestTracingContext(getFileSystem(), true)));
+ getTestTracingContext(getFileSystem(), true), null));
}
private AbfsRestOperationException getMockAbfsRestOperationException(int status) {
return new AbfsRestOperationException(status, "", "", new Exception());
}
+
+ /**
+ * Attempts to test multiple flush calls.
+ */
+ @Test
+ public void testMultipleFlush() throws Throwable {
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path testPath = new Path(TEST_FOLDER_PATH, TEST_CHILD_FILE);
+ try (FSDataOutputStream out = fs.create(testPath)) {
+ out.write('1');
+ out.hsync();
+ out.write('2');
+ out.hsync();
+ }
+ }
+
+ /**
+ * Delete the blob before flush and verify that an exception should be thrown.
+ */
+ @Test
+ public void testDeleteBeforeFlush() throws Throwable {
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path testPath = new Path(TEST_FOLDER_PATH, TEST_CHILD_FILE);
+ try (FSDataOutputStream out = fs.create(testPath)) {
+ out.write('1');
+ fs.delete(testPath, false);
+ out.hsync();
+ // this will cause the next write to failAll
+ } catch (IOException fnfe) {
+ //appendblob outputStream does not generate suppressed exception on close as it is
+ //single threaded code
+ if (!fs.getAbfsStore().isAppendBlobKey(fs.makeQualified(testPath).toString())) {
+ // the exception raised in close() must be in the caught exception's
+ // suppressed list
+ Throwable[] suppressed = fnfe.getSuppressed();
+ assertEquals("suppressed count", 1, suppressed.length);
+ Throwable inner = suppressed[0];
+ if (!(inner instanceof IOException)) {
+ throw inner;
+ }
+ GenericTestUtils.assertExceptionContains(fnfe.getMessage(), inner.getCause(), inner.getCause().getMessage());
+ }
+ }
+ }
+
+ /**
+ * Extracts the eTag for an existing file
+ * @param fileName file Path in String from container root
+ * @return String etag for the file
+ * @throws IOException
+ */
+ private String extractFileEtag(String fileName) throws IOException {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final AbfsClient client = fs.getAbfsClient();
+ final TracingContext testTracingContext = getTestTracingContext(fs, false);
+ AbfsRestOperation op;
+ if (fs.getAbfsStore().getPrefixMode() == PrefixMode.BLOB) {
+ op = client.getBlobProperty(new Path(fileName), testTracingContext);
+ } else {
+ op = client.getPathStatus(fileName, true, testTracingContext);
+ }
+ return AzureBlobFileSystemStore.extractEtagHeader(op.getResult());
+ }
}
+
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSASForBlobEndpoint.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSASForBlobEndpoint.java
new file mode 100644
index 00000000000000..32fdc78394779d
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSASForBlobEndpoint.java
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs;
+
+import java.util.List;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations;
+import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys;
+import org.apache.hadoop.fs.azurebfs.extensions.MockDelegationSASTokenProvider;
+import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
+import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation;
+import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation;
+import org.apache.hadoop.fs.azurebfs.services.AuthType;
+import org.apache.hadoop.fs.azurebfs.services.BlobProperty;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
+import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
+
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.COPY_STATUS_PENDING;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE;
+
+public class ITestAzureBlobFileSystemDelegationSASForBlobEndpoint
+ extends AbstractAbfsIntegrationTest {
+
+ public ITestAzureBlobFileSystemDelegationSASForBlobEndpoint()
+ throws Exception {
+ // These tests rely on specific settings in azure-auth-keys.xml:
+ String sasProvider = getRawConfiguration().get(
+ FS_AZURE_SAS_TOKEN_PROVIDER_TYPE);
+ Assume.assumeTrue(MockDelegationSASTokenProvider.class.getCanonicalName()
+ .equals(sasProvider));
+ Assume.assumeNotNull(
+ getRawConfiguration().get(TestConfigurationKeys.FS_AZURE_TEST_APP_ID));
+ Assume.assumeNotNull(getRawConfiguration().get(
+ TestConfigurationKeys.FS_AZURE_TEST_APP_SECRET));
+ Assume.assumeNotNull(getRawConfiguration().get(
+ TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_TENANT_ID));
+ Assume.assumeNotNull(getRawConfiguration().get(
+ TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_OBJECT_ID));
+ // The test uses shared key to create a random filesystem and then creates another
+ // instance of this filesystem using SAS authorization.
+ Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
+ }
+
+ @Override
+ public void setup() throws Exception {
+ createFilesystemForSASTests();
+ super.setup();
+ Assume.assumeTrue(
+ getFileSystem().getAbfsStore().getAbfsConfiguration().getPrefixMode()
+ == PrefixMode.BLOB);
+ }
+
+ @Before
+ public void before() throws Exception {
+ Assume.assumeTrue(
+ getFileSystem().getAbfsStore().getAbfsConfiguration().getPrefixMode()
+ == PrefixMode.BLOB);
+ }
+
+ @Test
+ public void testPosixRenameDirectoryWhereDirectoryAlreadyThereOnDestination()
+ throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ fs.mkdirs(new Path("testDir2/test1/test2/test3"));
+ fs.create(new Path("testDir2/test1/test2/test3/file"));
+ fs.mkdirs(new Path("testDir2/test4/test3"));
+ assertTrue(fs.exists(new Path("testDir2/test1/test2/test3/file")));
+ Assert.assertFalse(fs.rename(new Path("testDir2/test1/test2/test3"),
+ new Path("testDir2/test4")));
+ assertTrue(fs.exists(new Path("testDir2")));
+ assertTrue(fs.exists(new Path("testDir2/test1/test2")));
+ assertTrue(fs.exists(new Path("testDir2/test4")));
+ assertTrue(fs.exists(new Path("testDir2/test1/test2/test3")));
+ if (getIsNamespaceEnabled(fs) || fs.getAbfsStore()
+ .getAbfsConfiguration()
+ .get("fs.azure.abfs.account.name")
+ .contains(".dfs.core") || fs.getAbfsStore()
+ .getAbfsConfiguration()
+ .get("fs.azure.abfs.account.name")
+ .contains(".blob.core")) {
+ assertFalse(fs.exists(new Path("testDir2/test4/test3/file")));
+ assertTrue(fs.exists(new Path("testDir2/test1/test2/test3/file")));
+ } else {
+ assertTrue(fs.exists(new Path("testDir2/test4/test3/file")));
+ assertFalse(fs.exists(new Path("testDir2/test1/test2/test3/file")));
+ }
+ }
+
+ @Test
+ public void testPosixRenameDirectory() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ fs.mkdirs(new Path("testDir2/test1/test2/test3"));
+ fs.mkdirs(new Path("testDir2/test4"));
+ Assert.assertTrue(fs.rename(new Path("testDir2/test1/test2/test3"),
+ new Path("testDir2/test4")));
+ assertTrue(fs.exists(new Path("testDir2")));
+ assertTrue(fs.exists(new Path("testDir2/test1/test2")));
+ assertTrue(fs.exists(new Path("testDir2/test4")));
+ assertTrue(fs.exists(new Path("testDir2/test4/test3")));
+ assertFalse(fs.exists(new Path("testDir2/test1/test2/test3")));
+ }
+
+ @Test
+ public void testCopyBlobTakeTime() throws Exception {
+ AzureBlobFileSystem fileSystem = Mockito.spy(getFileSystem());
+ AzureBlobFileSystemStore store = Mockito.spy(fileSystem.getAbfsStore());
+ Mockito.doReturn(store).when(fileSystem).getAbfsStore();
+ AbfsClient client = store.getClient();
+ AbfsClient spiedClient = Mockito.spy(client);
+ store.setClient(spiedClient);
+
+ Mockito.doAnswer(answer -> {
+ AbfsRestOperation op = Mockito.spy((AbfsRestOperation) answer.callRealMethod());
+ AbfsHttpOperation httpOp = Mockito.spy(op.getResult());
+ Mockito.doReturn(COPY_STATUS_PENDING).when(httpOp).getResponseHeader(
+ HttpHeaderConfigurations.X_MS_COPY_STATUS);
+ Mockito.doReturn(httpOp).when(op).getResult();
+ return op;
+ }).when(spiedClient).copyBlob(Mockito.any(Path.class), Mockito.any(Path.class),
+ Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+ fileSystem.create(new Path("/test1/file"));
+ fileSystem.rename(new Path("/test1/file"), new Path("/test1/file2"));
+ Assert.assertTrue(fileSystem.exists(new Path("/test1/file2")));
+ Mockito.verify(store, Mockito.times(1))
+ .handleCopyInProgress(Mockito.any(Path.class),
+ Mockito.any(TracingContext.class), Mockito.any(String.class));
+ }
+
+ @Test
+ public void testListBlob() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ int i = 0;
+ while (i < 10) {
+ fs.create(new Path("/dir/" + i));
+ i++;
+ }
+ List blobProperties = fs.getAbfsStore()
+ .getListBlobs(new Path("dir"), null, null,
+ getTestTracingContext(fs, true), null, false);
+ Assertions.assertThat(blobProperties)
+ .describedAs(
+ "BlobList should match the number of files created in tests + the directory itself")
+ .hasSize(11);
+
+ blobProperties = fs.getAbfsStore()
+ .getListBlobs(new Path("dir"), null, null,
+ getTestTracingContext(fs, true), null, true);
+ Assertions.assertThat(blobProperties)
+ .describedAs(
+ "BlobList should match the number of files created in tests")
+ .hasSize(10);
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java
index 8afe3d4bca1b34..3bdbddd8dedcbb 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java
@@ -19,23 +19,35 @@
package org.apache.hadoop.fs.azurebfs;
import java.io.FileNotFoundException;
+import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.assertj.core.api.Assertions;
+import org.junit.Assert;
import org.junit.Assume;
+import org.junit.Ignore;
import org.junit.Test;
import org.apache.hadoop.fs.azurebfs.constants.FSOperationType;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
+import org.apache.hadoop.fs.azurebfs.services.AbfsClientTestUtil;
import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation;
import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation;
-import org.apache.hadoop.fs.azurebfs.services.TestAbfsClient;
+import org.apache.hadoop.fs.azurebfs.services.ListBlobProducer;
+import org.apache.hadoop.fs.azurebfs.services.ListBlobQueue;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
+import org.apache.hadoop.fs.azurebfs.services.ITestAbfsClient;
import org.apache.hadoop.fs.azurebfs.services.TestAbfsPerfTracker;
import org.apache.hadoop.fs.azurebfs.utils.TestMockHelpers;
import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
@@ -43,11 +55,19 @@
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.test.LambdaTestUtils;
+
+import org.mockito.Mockito;
import static java.net.HttpURLConnection.HTTP_BAD_REQUEST;
+import static java.net.HttpURLConnection.HTTP_FORBIDDEN;
import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
import static java.net.HttpURLConnection.HTTP_OK;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_MKDIRS_FALLBACK_TO_DFS;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_PRODUCER_QUEUE_MAX_SIZE;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_REDIRECT_DELETE;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_REDIRECT_RENAME;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doCallRealMethod;
import static org.mockito.Mockito.doReturn;
@@ -115,6 +135,20 @@ public void testEnsureFileIsDeleted() throws Exception {
assertPathDoesNotExist(fs, "deleted", testfile);
}
+ @Test
+ public void testEnsureFileIsDeletedWithRedirection() throws Exception {
+
+ // Set redirect to wasb delete as true and assert deletion.
+ Configuration configuration = Mockito.spy(getRawConfiguration());
+ configuration.setBoolean(FS_AZURE_REDIRECT_DELETE, true);
+ AzureBlobFileSystem fs1 = (AzureBlobFileSystem) FileSystem.newInstance(configuration);
+
+ Path testfile = makeQualified(new Path("testfile"));
+ touch(testfile);
+ assertDeleted(fs1, testfile, false);
+ assertPathDoesNotExist(fs1, "deleted", testfile);
+ }
+
@Test
public void testDeleteDirectory() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
@@ -127,6 +161,7 @@ public void testDeleteDirectory() throws Exception {
assertPathDoesNotExist(fs, "deleted", dir);
}
+ @Ignore
@Test
public void testDeleteFirstLevelDirectory() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
@@ -156,7 +191,7 @@ public Void call() throws Exception {
fs.getAbfsStore().getAbfsConfiguration().getClientCorrelationId(),
fs.getFileSystemId(), FSOperationType.DELETE, false, 0));
// first try a non-recursive delete, expect failure
- intercept(FileAlreadyExistsException.class,
+ intercept(IOException.class,
() -> fs.delete(dir, false));
fs.registerListener(null);
assertDeleted(fs, dir, true);
@@ -175,7 +210,7 @@ public void testDeleteIdempotency() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
AbfsClient abfsClient = fs.getAbfsStore().getClient();
- AbfsClient testClient = TestAbfsClient.createTestClientFromCurrentContext(
+ AbfsClient testClient = ITestAbfsClient.createTestClientFromCurrentContext(
abfsClient,
abfsConfig);
@@ -222,7 +257,7 @@ public void testDeleteIdempotency() throws Exception {
public void testDeleteIdempotencyTriggerHttp404() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
- AbfsClient client = TestAbfsClient.createTestClientFromCurrentContext(
+ AbfsClient client = ITestAbfsClient.createTestClientFromCurrentContext(
fs.getAbfsStore().getClient(),
this.getConfiguration());
@@ -241,7 +276,7 @@ public void testDeleteIdempotencyTriggerHttp404() throws Exception {
getTestTracingContext(fs, true)));
// mock idempotency check to mimic retried case
- AbfsClient mockClient = TestAbfsClient.getMockAbfsClient(
+ AbfsClient mockClient = ITestAbfsClient.getMockAbfsClient(
fs.getAbfsStore().getClient(),
this.getConfiguration());
AzureBlobFileSystemStore mockStore = mock(AzureBlobFileSystemStore.class);
@@ -256,10 +291,10 @@ public void testDeleteIdempotencyTriggerHttp404() throws Exception {
// Case 2: Mimic retried case
// Idempotency check on Delete always returns success
- AbfsRestOperation idempotencyRetOp = TestAbfsClient.getRestOp(
+ AbfsRestOperation idempotencyRetOp = ITestAbfsClient.getRestOp(
DeletePath, mockClient, HTTP_METHOD_DELETE,
- TestAbfsClient.getTestUrl(mockClient, "/NonExistingPath"),
- TestAbfsClient.getTestRequestHeaders(mockClient));
+ ITestAbfsClient.getTestUrl(mockClient, "/NonExistingPath"),
+ ITestAbfsClient.getTestRequestHeaders(mockClient));
idempotencyRetOp.hardSetResult(HTTP_OK);
doReturn(idempotencyRetOp).when(mockClient).deleteIdempotencyCheckOp(any());
@@ -282,4 +317,291 @@ public void testDeleteIdempotencyTriggerHttp404() throws Exception {
mockStore.delete(new Path("/NonExistingPath"), false, getTestTracingContext(fs, false));
}
+ @Test
+ public void testDeleteImplicitDir() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ Assume.assumeTrue(fs.getAbfsStore().getPrefixMode() == PrefixMode.BLOB);
+ fs.mkdirs(new Path("/testDir/dir1"));
+ fs.create(new Path("/testDir/dir1/file1"));
+ fs.getAbfsClient().deleteBlobPath(new Path("/testDir/dir1"),
+ null, getTestTracingContext(fs, true));
+
+ fs.delete(new Path("/testDir/dir1"), true);
+
+ Assert.assertTrue(!fs.exists(new Path("/testDir/dir1")));
+ Assert.assertTrue(!fs.exists(new Path("/testDir/dir1/file1")));
+ }
+
+ @Test
+ public void testDeleteImplicitDirWithSingleListResults() throws Exception {
+ AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.newInstance(
+ getRawConfiguration());
+ Assume.assumeTrue(fs.getAbfsStore().getPrefixMode() == PrefixMode.BLOB);
+ AbfsClient client = fs.getAbfsClient();
+ AbfsClient spiedClient = Mockito.spy(client);
+ fs.getAbfsStore().setClient(spiedClient);
+ fs.mkdirs(new Path("/testDir/dir1"));
+ for (int i = 0; i < 10; i++) {
+ fs.create(new Path("/testDir/dir1/file" + i));
+ }
+ Mockito.doAnswer(answer -> {
+ String marker = answer.getArgument(0);
+ String prefix = answer.getArgument(1);
+ TracingContext context = answer.getArgument(4);
+ return client.getListBlobs(marker, prefix, null, 1, context);
+ })
+ .when(spiedClient)
+ .getListBlobs(Mockito.nullable(String.class), Mockito.anyString(),
+ Mockito.nullable(String.class), Mockito.nullable(Integer.class),
+ Mockito.any(TracingContext.class));
+ fs.getAbfsClient().deleteBlobPath(new Path("/testDir/dir1"),
+ null, getTestTracingContext(fs, true));
+
+ fs.delete(new Path("/testDir/dir1"), true);
+
+ Assert.assertTrue(!fs.exists(new Path("/testDir/dir1")));
+ }
+
+ @Test
+ public void testDeleteExplicitDirInImplicitParentDir() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ Assume.assumeTrue(fs.getAbfsStore().getPrefixMode() == PrefixMode.BLOB);
+ fs.mkdirs(new Path("/testDir/dir1"));
+ fs.create(new Path("/testDir/dir1/file1"));
+ fs.getAbfsClient().deleteBlobPath(new Path("/testDir/"),
+ null, getTestTracingContext(fs, true));
+
+ fs.delete(new Path("/testDir/dir1"), true);
+
+ Assert.assertTrue(!fs.exists(new Path("/testDir/dir1")));
+ Assert.assertTrue(!fs.exists(new Path("/testDir/dir1/file1")));
+ Assert.assertTrue(fs.exists(new Path("/testDir/")));
+ }
+
+ @Test
+ public void testDeleteParallelBlobFailure() throws Exception {
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ Assume.assumeTrue(fs.getAbfsStore().getPrefixMode() == PrefixMode.BLOB);
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ store.setClient(client);
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+
+ fs.mkdirs(new Path("/testDir"));
+ fs.create(new Path("/testDir/file1"));
+ fs.create(new Path("/testDir/file2"));
+ fs.create(new Path("/testDir/file3"));
+
+ Mockito.doThrow(
+ new AbfsRestOperationException(HTTP_FORBIDDEN, "", "", new Exception()))
+ .when(client)
+ .deleteBlobPath(Mockito.any(Path.class), Mockito.nullable(String.class),
+ Mockito.any(TracingContext.class));
+
+ LambdaTestUtils.intercept(RuntimeException.class, () -> {
+ fs.delete(new Path("/testDir"), true);
+ });
+ }
+
+ @Test
+ public void testDeleteRootWithNonRecursion() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("/testDir"));
+ Assertions.assertThat(fs.delete(new Path("/"), false)).isFalse();
+ }
+
+ @Test
+ public void testDeleteCheckIfParentLMTChange() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ AbfsConfiguration conf = getConfiguration();
+ /*
+ * LMT of parent directory doesn't change when delete directory triggered with
+ * DFS endpoint (both hns and non-hns account). In Blob endpoint, if there
+ * is no redirect for ingress / mkdirs, the LMT doesn't change. But in case
+ * of ingress redirection, the directory creation of parent overrides the
+ * path which changes the LMT. Hence, for tests running with redirect
+ * configuration, this test is ignored.
+ */
+ Assume.assumeFalse(
+ getPrefixMode(fs) == PrefixMode.BLOB && (conf.shouldMkdirFallbackToDfs()
+ || conf.shouldIngressFallbackToDfs()));
+ fs.mkdirs(new Path("/dir1/dir2"));
+ fs.create(new Path("/dir1/dir2/file"));
+ FileStatus status = fs.getFileStatus(new Path("/dir1"));
+ Long lmt = status.getModificationTime();
+
+ fs.delete(new Path("/dir1/dir2"), true);
+ Long newLmt = fs.getFileStatus(new Path("/dir1")).getModificationTime();
+ Assertions.assertThat(lmt).isEqualTo(newLmt);
+ }
+
+ /**
+ * Test to assert that the CID in src marker delete contains the
+ * total number of blobs operated in the delete directory.
+ * Also, to assert that all operations in the delete-directory flow have same
+ * primaryId and opType.
+ */
+ @Test
+ public void testDeleteEmitDeletionCountInClientRequestId() throws Exception {
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ Assume.assumeTrue(getPrefixMode(fs) == PrefixMode.BLOB);
+ String dirPathStr = "/testDir/dir1";
+ fs.mkdirs(new Path(dirPathStr));
+ ExecutorService executorService = Executors.newFixedThreadPool(5);
+ List futures = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ final int iter = i;
+ Future future = executorService.submit(() -> {
+ return fs.create(new Path("/testDir/dir1/file" + iter));
+ });
+ futures.add(future);
+ }
+
+ for (Future future : futures) {
+ future.get();
+ }
+ executorService.shutdown();
+
+
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ AbfsClient client = Mockito.spy(store.getClient());
+ store.setClient(client);
+
+ final TracingHeaderValidator tracingHeaderValidator
+ = new TracingHeaderValidator(
+ fs.getAbfsStore().getAbfsConfiguration().getClientCorrelationId(),
+ fs.getFileSystemId(), FSOperationType.DELETE, true, 0);
+ fs.registerListener(tracingHeaderValidator);
+
+ Mockito.doAnswer(answer -> {
+ Mockito.doAnswer(deleteAnswer -> {
+ if (dirPathStr.equalsIgnoreCase(
+ ((Path) deleteAnswer.getArgument(0)).toUri().getPath())) {
+ tracingHeaderValidator.setOperatedBlobCount(11);
+ Object result = deleteAnswer.callRealMethod();
+ tracingHeaderValidator.setOperatedBlobCount(null);
+ return result;
+ }
+ return deleteAnswer.callRealMethod();
+ })
+ .when(client)
+ .deleteBlobPath(Mockito.any(Path.class),
+ Mockito.nullable(String.class),
+ Mockito.any(TracingContext.class));
+
+ return answer.callRealMethod();
+ })
+ .when(store)
+ .delete(Mockito.any(Path.class), Mockito.anyBoolean(),
+ Mockito.any(TracingContext.class));
+
+ fs.delete(new Path(dirPathStr), true);
+ }
+
+ @Test
+ public void testProducerStopOnDeleteFailure() throws Exception {
+ Assume.assumeTrue(getPrefixMode(getFileSystem()) == PrefixMode.BLOB);
+ Configuration configuration = Mockito.spy(getRawConfiguration());
+ AzureBlobFileSystem fs = Mockito.spy(
+ (AzureBlobFileSystem) FileSystem.get(configuration));
+
+ fs.mkdirs(new Path("/src"));
+ ExecutorService executorService = Executors.newFixedThreadPool(10);
+ List futureList = new ArrayList<>();
+ for (int i = 0; i < 20; i++) {
+ int iter = i;
+ Future future = executorService.submit(() -> {
+ try {
+ fs.create(new Path("/src/file" + iter));
+ } catch (IOException ex) {}
+ });
+ futureList.add(future);
+ }
+
+ for (Future future : futureList) {
+ future.get();
+ }
+
+ AbfsClient client = fs.getAbfsClient();
+ AbfsClient spiedClient = Mockito.spy(client);
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ store.setClient(spiedClient);
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+
+ ListBlobProducer[] producers = new ListBlobProducer[1];
+ Mockito.doAnswer(answer -> {
+ producers[0] = (ListBlobProducer) answer.callRealMethod();
+ return producers[0];
+ }).when(store).getListBlobProducer(Mockito.anyString(), Mockito.any(
+ ListBlobQueue.class), Mockito.nullable(String.class),
+ Mockito.any(TracingContext.class));
+
+ AtomicInteger listCounter = new AtomicInteger(0);
+ AtomicBoolean hasConsumerStarted = new AtomicBoolean(false);
+
+ Mockito.doAnswer(answer -> {
+ String marker = answer.getArgument(0);
+ String prefix = answer.getArgument(1);
+ String delimiter = answer.getArgument(2);
+ TracingContext tracingContext = answer.getArgument(4);
+ int counter = listCounter.incrementAndGet();
+ if (counter > 1) {
+ while (!hasConsumerStarted.get()) {
+ Thread.sleep(1_000L);
+ }
+ }
+ Object result = client.getListBlobs(marker, prefix, delimiter, 1,
+ tracingContext);
+ return result;
+ })
+ .when(spiedClient)
+ .getListBlobs(Mockito.nullable(String.class),
+ Mockito.nullable(String.class), Mockito.nullable(String.class),
+ Mockito.nullable(Integer.class), Mockito.any(TracingContext.class));
+
+ Mockito.doAnswer(answer -> {
+ spiedClient.acquireBlobLease(
+ ((Path) answer.getArgument(0)).toUri().getPath(), -1,
+ answer.getArgument(2));
+ hasConsumerStarted.set(true);
+ return answer.callRealMethod();
+ })
+ .when(spiedClient)
+ .deleteBlobPath(Mockito.any(Path.class), Mockito.nullable(String.class),
+ Mockito.any(TracingContext.class));
+
+ intercept(Exception.class, () -> {
+ fs.delete(new Path("/src"), true);
+ });
+
+ producers[0].waitForProcessCompletion();
+
+ Mockito.verify(spiedClient, Mockito.atMost(3))
+ .getListBlobs(Mockito.nullable(String.class),
+ Mockito.nullable(String.class), Mockito.nullable(String.class),
+ Mockito.nullable(Integer.class), Mockito.any(TracingContext.class));
+ }
+
+ @Test
+ public void deleteBlobDirParallelThreadToDeleteOnDifferentTracingContext()
+ throws Exception {
+ Assume.assumeTrue(getPrefixMode(getFileSystem()) == PrefixMode.BLOB);
+ Configuration configuration = getRawConfiguration();
+ AzureBlobFileSystem fs = Mockito.spy(
+ (AzureBlobFileSystem) FileSystem.newInstance(configuration));
+ AzureBlobFileSystemStore spiedStore = Mockito.spy(fs.getAbfsStore());
+ AbfsClient spiedClient = Mockito.spy(fs.getAbfsClient());
+
+ Mockito.doReturn(spiedStore).when(fs).getAbfsStore();
+ spiedStore.setClient(spiedClient);
+
+ fs.mkdirs(new Path("/testDir"));
+ fs.create(new Path("/testDir/file1"));
+ fs.create(new Path("/testDir/file2"));
+
+ AbfsClientTestUtil.hookOnRestOpsForTracingContextSingularity(spiedClient);
+
+ fs.delete(new Path("/testDir"), true);
+ }
}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java
index 05c3855f5c89de..e525e709affbdd 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java
@@ -18,11 +18,14 @@
package org.apache.hadoop.fs.azurebfs;
+import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Arrays;
import java.util.Random;
+import org.apache.hadoop.fs.azurebfs.services.OperativeEndpoint;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
@@ -195,7 +198,7 @@ public void testWriteWithFileNotFoundException() throws Exception {
assertFalse(fs.exists(testFilePath));
// trigger append call
- intercept(FileNotFoundException.class,
+ intercept(IOException.class,
() -> stream.close());
}
@@ -213,8 +216,13 @@ public void testFlushWithFileNotFoundException() throws Exception {
fs.delete(testFilePath, true);
assertFalse(fs.exists(testFilePath));
- intercept(FileNotFoundException.class,
- () -> stream.close());
+ // No call goes to backend for empty map in case of blob endpoint.
+ if (OperativeEndpoint.isIngressEnabledOnDFS(getPrefixMode(fs), fs.getAbfsStore().getAbfsConfiguration())) {
+ intercept(FileNotFoundException.class,
+ () -> stream.close());
+ } else {
+ stream.close();
+ }
}
private void testWriteOneByteToFile(Path testFilePath) throws Exception {
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemExplictImplicitRename.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemExplictImplicitRename.java
new file mode 100644
index 00000000000000..f597e3f3419215
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemExplictImplicitRename.java
@@ -0,0 +1,964 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
+import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+public class ITestAzureBlobFileSystemExplictImplicitRename
+ extends AbstractAbfsIntegrationTest {
+
+ public ITestAzureBlobFileSystemExplictImplicitRename() throws Exception {
+ super();
+ }
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ Assume.assumeTrue(
+ getFileSystem().getAbfsStore().getAbfsConfiguration().getPrefixMode()
+ == PrefixMode.BLOB);
+ }
+
+ @Test
+ public void testRenameSrcFileInImplicitParentDirectory() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ createAzCopyDirectory(new Path("/src"));
+ createAzCopyFile(new Path("/src/file"));
+ intercept(AbfsRestOperationException.class, () -> {
+ fs.getAbfsStore().getBlobProperty(new Path("/src"), getTestTracingContext(fs, true));
+ });
+ Assert.assertNotNull(fs.getAbfsStore()
+ .getBlobProperty(new Path("/src/file"),
+ getTestTracingContext(fs, true)));
+ Assert.assertTrue(fs.rename(new Path("/src/file"), new Path("/dstFile")));
+ Assert.assertNotNull(fs.getAbfsStore()
+ .getBlobProperty(new Path("/dstFile"),
+ getTestTracingContext(fs, true)));
+ intercept(AbfsRestOperationException.class, () -> {
+ fs.getAbfsStore()
+ .getBlobProperty(new Path("/src/file"),
+ getTestTracingContext(fs, true));
+ });
+
+ Assert.assertFalse(fs.rename(new Path("/src/file"), new Path("/dstFile2")));
+ }
+
+ @Test
+ public void testRenameNonExistentFileInImplicitParent() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ createAzCopyDirectory(new Path("/src"));
+ intercept(AbfsRestOperationException.class, () -> {
+ fs.getAbfsStore().getBlobProperty(new Path("/src"), getTestTracingContext(fs, true));
+ });
+
+ Assert.assertFalse(fs.rename(new Path("/src/file"), new Path("/dstFile2")));
+ }
+
+ @Test
+ public void testRenameFileToNonExistingDstInImplicitParent()
+ throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ createAzCopyFile(new Path("/file"));
+ createAzCopyDirectory(new Path("/dstDir"));
+ createAzCopyFile(new Path("/dstDir/file2"));
+ intercept(AbfsRestOperationException.class, () -> {
+ fs.getAbfsStore()
+ .getBlobProperty(new Path("/dstDir"),
+ getTestTracingContext(fs, true));
+ });
+ Assert.assertTrue(fs.rename(new Path("/file"), new Path("/dstDir")));
+ Assert.assertTrue(fs.exists(new Path("/dstDir/file")));
+ }
+
+ @Test
+ public void testRenameFileAsExistingExplicitDirectoryInImplicitDirectory()
+ throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ createAzCopyFile(new Path("/file"));
+ createAzCopyDirectory(new Path("/dst"));
+ fs.mkdirs(new Path("/dst/dir"));
+ deleteBlobPath(fs, new Path("/dst"));
+ intercept(AbfsRestOperationException.class, () -> {
+ fs.getAbfsStore()
+ .getBlobProperty(new Path("/dst"),
+ getTestTracingContext(fs, true));
+ });
+ Assert.assertTrue(fs.rename(new Path("/file"), new Path("/dst/dir")));
+ Assert.assertTrue(fs.exists(new Path("/dst/dir/file")));
+ intercept(AbfsRestOperationException.class, () -> {
+ fs.getAbfsStore()
+ .getBlobProperty(new Path("/file"),
+ getTestTracingContext(fs, true));
+ });
+ }
+
+ @Test
+ public void testRenameFileAsExistingImplicitDirectoryInExplicitDirectory()
+ throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ createAzCopyFile(new Path("/file"));
+ fs.mkdirs(new Path("/dst"));
+ createAzCopyDirectory(new Path("/dst/dir"));
+ createAzCopyFile(new Path("/dst/dir/file2"));
+ intercept(AbfsRestOperationException.class, () -> {
+ fs.getAbfsStore()
+ .getBlobProperty(new Path("/dst/dir"),
+ getTestTracingContext(fs, true));
+ });
+ Assert.assertTrue(fs.rename(new Path("/file"), new Path("/dst/dir")));
+ Assert.assertTrue(fs.exists(new Path("/dst/dir/file")));
+ intercept(AbfsRestOperationException.class, () -> {
+ fs.getAbfsStore()
+ .getBlobProperty(new Path("/file"),
+ getTestTracingContext(fs, true));
+ });
+ }
+
+ @Test
+ public void testRenameFileAsExistingImplicitDirectoryInImplicitDirectory()
+ throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ createAzCopyFile(new Path("/file"));
+ createAzCopyDirectory(new Path("/dst"));
+ createAzCopyDirectory(new Path("/dst/dir"));
+ createAzCopyFile(new Path("/dst/dir/file2"));
+ intercept(AbfsRestOperationException.class, () -> {
+ fs.getAbfsStore()
+ .getBlobProperty(new Path("/dst"),
+ getTestTracingContext(fs, true));
+ });
+ intercept(AbfsRestOperationException.class, () -> {
+ fs.getAbfsStore()
+ .getBlobProperty(new Path("/dst/dir"),
+ getTestTracingContext(fs, true));
+ });
+ Assert.assertTrue(fs.rename(new Path("/file"), new Path("/dst/dir")));
+ Assert.assertTrue(fs.exists(new Path("/dst/dir/file")));
+ intercept(AbfsRestOperationException.class, () -> {
+ fs.getAbfsStore()
+ .getBlobProperty(new Path("/file"),
+ getTestTracingContext(fs, true));
+ });
+ }
+
+ @Test
+ public void testRenameDirectoryContainingImplicitDirectory()
+ throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("/src"));
+ fs.mkdirs(new Path("/dst"));
+ createAzCopyDirectory(new Path("/src/subDir"));
+ createAzCopyFile(new Path("/src/subDir/subFile"));
+ createAzCopyFile(new Path("/src/subFile"));
+ Assert.assertTrue(fs.rename(new Path("/src"), new Path("/dst/dir")));
+ Assert.assertTrue(fs.exists(new Path("/dst/dir/subFile")));
+ Assert.assertTrue(fs.exists(new Path("/dst/dir/subDir/subFile")));
+ }
+
+ @Test
+ public void testRenameImplicitDirectoryContainingExplicitDirectory()
+ throws Exception {
+ explicitImplicitDirectoryRenameTest(
+ /*srcParentExplicit*/true,
+ /*srcExplicit*/false,
+ /*srcSubDirExplicit*/true,
+ /*dstParentExplicit*/true,
+ /*dstExplicit*/false,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/false,
+ /*isDstFile*/false,
+ /*shouldRenamePass*/true
+ );
+ }
+
+ @Test
+ public void testRenameImplicitDirectoryContainingImplicitDirectory()
+ throws Exception {
+ explicitImplicitDirectoryRenameTest(
+ /*srcParentExplicit*/true,
+ /*srcExplicit*/false,
+ /*srcSubDirExplicit*/false,
+ /*dstParentExplicit*/true,
+ /*dstExplicit*/false,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/false,
+ /*isDstFile*/false,
+ /*shouldRenamePass*/true
+ );
+ }
+
+ @Test
+ public void testRenameExplicitDirectoryContainingExplicitDirectoryInImplicitSrcParent()
+ throws Exception {
+ explicitImplicitDirectoryRenameTest(
+ /*srcParentExplicit*/false,
+ /*srcExplicit*/true,
+ /*srcSubDirExplicit*/true,
+ /*dstParentExplicit*/true,
+ /*dstExplicit*/false,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/false,
+ /*isDstFile*/false,
+ /*shouldRenamePass*/true
+ );
+ }
+
+ @Test
+ public void testRenameExplicitDirectoryContainingImplicitDirectoryInImplicitSrcParent()
+ throws Exception {
+ explicitImplicitDirectoryRenameTest(
+ /*srcParentExplicit*/false,
+ /*srcExplicit*/true,
+ /*srcSubDirExplicit*/false,
+ /*dstParentExplicit*/true,
+ /*dstExplicit*/false,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/false,
+ /*isDstFile*/false,
+ /*shouldRenamePass*/true
+ );
+ }
+
+ @Test
+ public void testRenameImplicitDirectoryContainingExplicitDirectoryInImplicitSrcParent()
+ throws Exception {
+ explicitImplicitDirectoryRenameTest(
+ /*srcParentExplicit*/false,
+ /*srcExplicit*/false,
+ /*srcSubDirExplicit*/true,
+ /*dstParentExplicit*/true,
+ /*dstExplicit*/false,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/false,
+ /*isDstFile*/false,
+ /*shouldRenamePass*/true
+ );
+ }
+
+ @Test
+ public void testRenameImplicitDirectoryContainingImplicitDirectoryInImplicitSrcParent()
+ throws Exception {
+ explicitImplicitDirectoryRenameTest(
+ /*srcParentExplicit*/false,
+ /*srcExplicit*/false,
+ /*srcSubDirExplicit*/false,
+ /*dstParentExplicit*/true,
+ /*dstExplicit*/false,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/false,
+ /*isDstFile*/false,
+ /*shouldRenamePass*/true
+ );
+ }
+
+ @Test
+ public void testRenameDirectoryWhereDstParentDoesntExist() throws Exception {
+ explicitImplicitDirectoryRenameTest(
+ /*srcParentExplicit*/true,
+ /*srcExplicit*/true,
+ /*srcSubDirExplicit*/false,
+ /*dstParentExplicit*/false,
+ /*dstExplicit*/false,
+ /*dstParentExists*/false,
+ /*isDstParentFile*/false,
+ /*dstExist*/false,
+ /*isDstFile*/false,
+ /*shouldRenamePass*/false
+ );
+ }
+
+ @Test
+ public void testRenameImplicitDirectoryWhereDstParentDoesntExist()
+ throws Exception {
+ explicitImplicitDirectoryRenameTest(
+ /*srcParentExplicit*/true,
+ /*srcExplicit*/false,
+ /*srcSubDirExplicit*/true,
+ /*dstParentExplicit*/false,
+ /*dstExplicit*/false,
+ /*dstParentExists*/false,
+ /*isDstParentFile*/false,
+ /*dstExist*/false,
+ /*isDstFile*/false,
+ /*shouldRenamePass*/false
+ );
+ }
+
+ @Test
+ public void testRenameImplicitDirectoryToNonExistentDstWithImplicitParent()
+ throws Exception {
+ explicitImplicitDirectoryRenameTest(
+ /*srcParentExplicit*/true,
+ /*srcExplicit*/false,
+ /*srcSubDirExplicit*/true,
+ /*dstParentExplicit*/false,
+ /*dstExplicit*/false,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/false,
+ /*isDstFile*/false,
+ /*shouldRenamePass*/true
+ );
+ }
+
+ @Test
+ public void testRenameImplicitDirectoryToNonExistentDstWithParentIsFile()
+ throws Exception {
+ explicitImplicitDirectoryRenameTest(
+ /*srcParentExplicit*/true,
+ /*srcExplicit*/false,
+ /*srcSubDirExplicit*/true,
+ /*dstParentExplicit*/false,
+ /*dstExplicit*/false,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/true,
+ /*dstExist*/false,
+ /*isDstFile*/false,
+ /*shouldRenamePass*/false
+ );
+ }
+
+ @Test
+ public void testRenameExplicitDirectoryToFileDst() throws Exception {
+ explicitImplicitDirectoryRenameTest(
+ /*srcParentExplicit*/true,
+ /*srcExplicit*/true,
+ /*srcSubDirExplicit*/true,
+ /*dstParentExplicit*/true,
+ /*dstExplicit*/false,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/true,
+ /*isDstFile*/true,
+ /*shouldRenamePass*/false
+ );
+ }
+
+ @Test
+ public void testRenameimplicitDirectoryToFileDst() throws Exception {
+ explicitImplicitDirectoryRenameTest(
+ /*srcParentExplicit*/true,
+ /*srcExplicit*/false,
+ /*srcSubDirExplicit*/true,
+ /*dstParentExplicit*/true,
+ /*dstExplicit*/false,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/true,
+ /*isDstFile*/true,
+ /*shouldRenamePass*/false
+ );
+ }
+
+ @Test
+ public void testDirectoryIntoSameNameDestination() throws Exception {
+ explicitImplicitDirectoryRenameTestWithDestPathNames(
+ /*srcParentExplicit*/true,
+ /*srcExplicit*/true,
+ /*srcSubDirExplicit*/true,
+ /*dstParentExplicit*/true,
+ /*dstExplicit*/true,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/true,
+ /*isDstFile*/false,
+ /*srcName*/"src",
+ /*dstName*/"src",
+ /*dstSubFileName*/null,
+ /*dstSubDirName*/null,
+ /*isSubDirExplicit*/true, /*shouldRenamePass*/true
+ );
+ }
+
+ @Test
+ public void testRenameDirectoryToSameNameImplicitDirectoryDestination()
+ throws Exception {
+ explicitImplicitDirectoryRenameTestWithDestPathNames(
+ /*srcParentExplicit*/true,
+ /*srcExplicit*/true,
+ /*srcSubDirExplicit*/true,
+ /*dstParentExplicit*/true,
+ /*dstExplicit*/false,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/true,
+ /*isDstFile*/false,
+ /*srcName*/"src",
+ /*dstName*/"src",
+ /*dstSubFileName*/null,
+ /*dstSubDirName*/null,
+ /*isSubDirExplicit*/true, /*shouldRenamePass*/true
+ );
+ }
+
+ @Test
+ public void testRenameDirectoryToImplicitDirectoryDestinationHavingSameNameSubDir()
+ throws Exception {
+ explicitImplicitDirectoryRenameTestWithDestPathNames(
+ /*srcParentExplicit*/true,
+ /*srcExplicit*/true,
+ /*srcSubDirExplicit*/true,
+ /*dstParentExplicit*/true,
+ /*dstExplicit*/false,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/true,
+ /*isDstFile*/false,
+ /*srcName*/"src",
+ /*dstName*/null,
+ /*dstSubFileName*/null,
+ /*dstSubDirName*/"src",
+ /*isSubDirExplicit*/true, /*shouldRenamePass*/false
+ );
+ }
+
+ @Test
+ public void testRenameDirectoryToImplicitDirectoryDestinationHavingSameNameSubFile()
+ throws Exception {
+ explicitImplicitDirectoryRenameTestWithDestPathNames(
+ /*srcParentExplicit*/true,
+ /*srcExplicit*/true,
+ /*srcSubDirExplicit*/true,
+ /*dstParentExplicit*/true,
+ /*dstExplicit*/false,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/true,
+ /*isDstFile*/false,
+ /*srcName*/"src",
+ /*dstName*/null,
+ /*dstSubFileName*/"src",
+ /*dstSubDirName*/null,
+ /*isSubDirExplicit*/true, /*shouldRenamePass*/false
+ );
+ }
+
+ @Test
+ public void testRenameDirectoryToImplicitDirectoryDestinationHavingSameNameImplicitSubDir()
+ throws Exception {
+ explicitImplicitDirectoryRenameTestWithDestPathNames(
+ /*srcParentExplicit*/true,
+ /*srcExplicit*/true,
+ /*srcSubDirExplicit*/true,
+ /*dstParentExplicit*/true,
+ /*dstExplicit*/false,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/true,
+ /*isDstFile*/false,
+ /*srcName*/"src",
+ /*dstName*/null,
+ /*dstSubFileName*/null,
+ /*dstSubDirName*/"src",
+ /*isSubDirExplicit*/false, /*shouldRenamePass*/false
+ );
+ }
+
+ @Test
+ public void testImplicitDirectoryIntoSameNameDestination() throws Exception {
+ explicitImplicitDirectoryRenameTestWithDestPathNames(
+ /*srcParentExplicit*/true,
+ /*srcExplicit*/false,
+ /*srcSubDirExplicit*/true,
+ /*dstParentExplicit*/true,
+ /*dstExplicit*/true,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/true,
+ /*isDstFile*/false,
+ /*srcName*/"src",
+ /*dstName*/"src",
+ /*dstSubFileName*/null,
+ /*dstSubDirName*/null,
+ /*isSubDirExplicit*/true, /*shouldRenamePass*/true
+ );
+ }
+
+ @Test
+ public void testImplicitDirectoryIntoExplicitDestination() throws Exception {
+ explicitImplicitDirectoryRenameTestWithDestPathNames(
+ /*srcParentExplicit*/true,
+ /*srcExplicit*/false,
+ /*srcSubDirExplicit*/true,
+ /*dstParentExplicit*/true,
+ /*dstExplicit*/true,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/true,
+ /*isDstFile*/false,
+ /*srcName*/"src",
+ /*dstName*/null,
+ /*dstSubFileName*/null,
+ /*dstSubDirName*/null,
+ /*isSubDirExplicit*/true, /*shouldRenamePass*/true
+ );
+ }
+
+ @Test
+ public void testRenameImplicitDirectoryToExplicitDirectoryDestinationHavingSameNameSubDir()
+ throws Exception {
+ explicitImplicitDirectoryRenameTestWithDestPathNames(
+ /*srcParentExplicit*/true,
+ /*srcExplicit*/false,
+ /*srcSubDirExplicit*/true,
+ /*dstParentExplicit*/true,
+ /*dstExplicit*/true,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/true,
+ /*isDstFile*/false,
+ /*srcName*/"src",
+ /*dstName*/null,
+ /*dstSubFileName*/null,
+ /*dstSubDirName*/"src",
+ /*isSubDirExplicit*/true, /*shouldRenamePass*/false
+ );
+ }
+
+ @Test
+ public void testRenameImplicitDirectoryToExplicitDirectoryDestinationHavingSameNameSubFile()
+ throws Exception {
+ explicitImplicitDirectoryRenameTestWithDestPathNames(
+ /*srcParentExplicit*/true,
+ /*srcExplicit*/false,
+ /*srcSubDirExplicit*/true,
+ /*dstParentExplicit*/true,
+ /*dstExplicit*/true,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/true,
+ /*isDstFile*/false,
+ /*srcName*/"src",
+ /*dstName*/null,
+ /*dstSubFileName*/"src",
+ /*dstSubDirName*/null,
+ /*isSubDirExplicit*/true, /*shouldRenamePass*/false
+ );
+ }
+
+ @Test
+ public void testRenameImplicitDirectoryToExplicitDirectoryDestinationHavingSameNameImplicitSubDir()
+ throws Exception {
+ explicitImplicitDirectoryRenameTestWithDestPathNames(
+ /*srcParentExplicit*/true,
+ /*srcExplicit*/false,
+ /*srcSubDirExplicit*/true,
+ /*dstParentExplicit*/true,
+ /*dstExplicit*/true,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/true,
+ /*isDstFile*/false,
+ /*srcName*/"src",
+ /*dstName*/null,
+ /*dstSubFileName*/null,
+ /*dstSubDirName*/"src",
+ /*isSubDirExplicit*/false, /*shouldRenamePass*/false
+ );
+ }
+
+ @Test
+ public void testImplicitDirectoryIntoSameNameImplicitDestination()
+ throws Exception {
+ explicitImplicitDirectoryRenameTestWithDestPathNames(
+ /*srcParentExplicit*/true,
+ /*srcExplicit*/false,
+ /*srcSubDirExplicit*/true,
+ /*dstParentExplicit*/true,
+ /*dstExplicit*/false,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/true,
+ /*isDstFile*/false,
+ /*srcName*/"src",
+ /*dstName*/"src",
+ /*dstSubFileName*/null,
+ /*dstSubDirName*/null,
+ /*isSubDirExplicit*/true, /*shouldRenamePass*/true
+ );
+ }
+
+ @Test
+ public void testImplicitDirectoryIntoImplicitDestination() throws Exception {
+ explicitImplicitDirectoryRenameTestWithDestPathNames(
+ /*srcParentExplicit*/true,
+ /*srcExplicit*/false,
+ /*srcSubDirExplicit*/true,
+ /*dstParentExplicit*/true,
+ /*dstExplicit*/false,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/true,
+ /*isDstFile*/false,
+ /*srcName*/"src",
+ /*dstName*/null,
+ /*dstSubFileName*/null,
+ /*dstSubDirName*/null,
+ /*isSubDirExplicit*/true, /*shouldRenamePass*/true
+ );
+ }
+
+ @Test
+ public void testRenameImplicitDirectoryToImplicitDirectoryDestinationHavingSameNameSubDir()
+ throws Exception {
+ explicitImplicitDirectoryRenameTestWithDestPathNames(
+ /*srcParentExplicit*/true,
+ /*srcExplicit*/false,
+ /*srcSubDirExplicit*/true,
+ /*dstParentExplicit*/true,
+ /*dstExplicit*/false,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/true,
+ /*isDstFile*/false,
+ /*srcName*/"src",
+ /*dstName*/null,
+ /*dstSubFileName*/null,
+ /*dstSubDirName*/"src",
+ /*isSubDirExplicit*/true, /*shouldRenamePass*/false
+ );
+ }
+
+ @Test
+ public void testRenameImplicitDirectoryToImplicitDirectoryDestinationHavingSameNameSubFile()
+ throws Exception {
+ explicitImplicitDirectoryRenameTestWithDestPathNames(
+ /*srcParentExplicit*/true,
+ /*srcExplicit*/false,
+ /*srcSubDirExplicit*/true,
+ /*dstParentExplicit*/true,
+ /*dstExplicit*/false,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/true,
+ /*isDstFile*/false,
+ /*srcName*/"src",
+ /*dstName*/null,
+ /*dstSubFileName*/"src",
+ /*dstSubDirName*/null,
+ /*isSubDirExplicit*/true, /*shouldRenamePass*/false
+ );
+ }
+
+ @Test
+ public void testRenameImplicitDirectoryToImplicitDirectoryDestinationHavingSameNameImplicitSubDir()
+ throws Exception {
+ explicitImplicitDirectoryRenameTestWithDestPathNames(
+ /*srcParentExplicit*/true,
+ /*srcExplicit*/false,
+ /*srcSubDirExplicit*/true,
+ /*dstParentExplicit*/true,
+ /*dstExplicit*/false,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/true,
+ /*isDstFile*/false,
+ /*srcName*/"src",
+ /*dstName*/null,
+ /*dstSubFileName*/null,
+ /*dstSubDirName*/"src",
+ /*isSubDirExplicit*/false, /*shouldRenamePass*/false
+ );
+ }
+
+ @Test
+ public void testRenameExplicitSrcWithImplicitSubDirToImplicitDstWithExplicitSubDir()
+ throws Exception {
+ explicitImplicitDirectoryRenameTestWithDestPathNames(
+ /*srcParentExplicit*/true,
+ /*srcExplicit*/true,
+ /*srcSubDirExplicit*/false,
+ /*dstParentExplicit*/false,
+ /*dstExplicit*/true,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/true,
+ /*isDstFile*/false,
+ /*srcName*/"src",
+ /*dstName*/null,
+ /*dstSubFileName*/null,
+ /*dstSubDirName*/null,
+ /*isSubDirExplicit*/true, /*shouldRenamePass*/true
+ );
+ }
+
+ @Test
+ public void testRenameExplicitSrcWithImplicitSubDirToImplicitDstWithImplicitSubDir()
+ throws Exception {
+ explicitImplicitDirectoryRenameTestWithDestPathNames(
+ /*srcParentExplicit*/true,
+ /*srcExplicit*/true,
+ /*srcSubDirExplicit*/false,
+ /*dstParentExplicit*/false,
+ /*dstExplicit*/true,
+ /*dstParentExists*/true,
+ /*isDstParentFile*/false,
+ /*dstExist*/true,
+ /*isDstFile*/false,
+ /*srcName*/"src",
+ /*dstName*/null,
+ /*dstSubFileName*/null,
+ /*dstSubDirName*/null,
+ /*isSubDirExplicit*/false, /*shouldRenamePass*/true
+ );
+ }
+
+
+ private void explicitImplicitDirectoryRenameTest(Boolean srcParentExplicit,
+ Boolean srcExplicit,
+ Boolean srcSubDirExplicit,
+ Boolean dstParentExplicit,
+ Boolean dstExplicit,
+ Boolean dstParentExists,
+ Boolean isDstParentFile,
+ Boolean dstExist,
+ Boolean isDstFile,
+ Boolean shouldRenamePass) throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ Path srcParent = new Path("/srcParent");
+ Path src = new Path(srcParent, "src");
+ createSourcePaths(srcParentExplicit, srcExplicit, srcSubDirExplicit, fs,
+ srcParent,
+ src);
+
+ Path dstParent = new Path("/dstParent");
+ Path dst = new Path(dstParent, "dst");
+ createDestinationPaths(dstParentExplicit, dstExplicit, dstParentExists,
+ isDstParentFile,
+ dstExist, isDstFile, fs, dstParent, dst, null, null, true);
+
+ if (dstParentExists && !isDstParentFile && !dstParentExplicit) {
+ intercept(AbfsRestOperationException.class, () -> {
+ fs.getAbfsStore()
+ .getBlobProperty(dstParent, getTestTracingContext(fs, true));
+ });
+ }
+
+ explicitImplicitCaseRenameAssert(dstExist, shouldRenamePass, fs, src, dst);
+ }
+
+ private void explicitImplicitDirectoryRenameTestWithDestPathNames(Boolean srcParentExplicit,
+ Boolean srcExplicit,
+ Boolean srcSubDirExplicit,
+ Boolean dstParentExplicit,
+ Boolean dstExplicit,
+ Boolean dstParentExists,
+ Boolean isDstParentFile,
+ Boolean dstExist,
+ Boolean isDstFile,
+ String srcName,
+ String dstName,
+ String dstSubFileName,
+ String dstSubDirName,
+ final Boolean isSubDirExplicit, Boolean shouldRenamePass)
+ throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ Path srcParent = new Path("/srcParent");
+ Path src = new Path(srcParent, srcName != null ? srcName : "src");
+ createSourcePaths(srcParentExplicit, srcExplicit, srcSubDirExplicit, fs,
+ srcParent,
+ src);
+
+ Path dstParent = new Path("/dstParent");
+ Path dst = new Path(dstParent, dstName != null ? dstName : "dst");
+ createDestinationPaths(dstParentExplicit, dstExplicit, dstParentExists,
+ isDstParentFile,
+ dstExist, isDstFile, fs, dstParent, dst, dstSubFileName, dstSubDirName,
+ isSubDirExplicit);
+
+ if (dstParentExists && !isDstParentFile && !dstParentExplicit && !dstExplicit) {
+ intercept(AbfsRestOperationException.class, () -> {
+ fs.getAbfsStore()
+ .getBlobProperty(dstParent, getTestTracingContext(fs, true));
+ });
+ }
+
+ explicitImplicitCaseRenameAssert(dstExist, shouldRenamePass, fs, src, dst);
+ }
+
+ private void createSourcePaths(final Boolean srcParentExplicit,
+ final Boolean srcExplicit,
+ final Boolean srcSubDirExplicit,
+ final AzureBlobFileSystem fs,
+ final Path srcParent,
+ final Path src) throws Exception {
+ if (srcParentExplicit) {
+ fs.mkdirs(srcParent);
+ } else {
+ createAzCopyDirectory(srcParent);
+ }
+
+ if (srcExplicit) {
+ fs.mkdirs(src);
+ if(!srcParentExplicit) {
+ deleteBlobPath(fs, srcParent);
+ }
+ } else {
+ createAzCopyDirectory(src);
+ }
+ createAzCopyFile(new Path(src, "subFile"));
+ if (srcSubDirExplicit) {
+ fs.mkdirs(new Path(src, "subDir"));
+ if(!srcParentExplicit) {
+ deleteBlobPath(fs, srcParent);
+ }
+ if(!srcExplicit) {
+ deleteBlobPath(fs, src);
+ }
+ } else {
+ Path srcSubDir = new Path(src, "subDir");
+ createAzCopyDirectory(srcSubDir);
+ createAzCopyFile(new Path(srcSubDir, "subFile"));
+ intercept(AbfsRestOperationException.class, () -> {
+ fs.getAbfsStore()
+ .getBlobProperty(srcSubDir, getTestTracingContext(fs, true));
+ });
+ }
+ if (!srcParentExplicit) {
+ intercept(AbfsRestOperationException.class, () -> {
+ fs.getAbfsStore()
+ .getBlobProperty(srcParent, getTestTracingContext(fs, true));
+ });
+ }
+ if (!srcExplicit) {
+ intercept(AbfsRestOperationException.class, () -> {
+ fs.getAbfsStore()
+ .getBlobProperty(src, getTestTracingContext(fs, true));
+ });
+ }
+ }
+
+ private void deleteBlobPath(final AzureBlobFileSystem fs, final Path srcParent)
+ throws AzureBlobFileSystemException {
+ try {
+ fs.getAbfsClient()
+ .deleteBlobPath(srcParent, null, getTestTracingContext(fs, true));
+ } catch (AbfsRestOperationException ex) {
+ if(ex.getStatusCode() != HttpURLConnection.HTTP_NOT_FOUND) {
+ throw ex;
+ }
+ }
+ }
+
+ private void createDestinationPaths(final Boolean dstParentExplicit,
+ final Boolean dstExplicit,
+ final Boolean dstParentExists,
+ final Boolean isDstParentFile,
+ final Boolean dstExist,
+ final Boolean isDstFile,
+ final AzureBlobFileSystem fs,
+ final Path dstParent,
+ final Path dst, final String subFileName, final String subDirName,
+ final Boolean isSubDirExplicit) throws Exception {
+ if (dstParentExists) {
+ if (!isDstParentFile) {
+ if (dstParentExplicit) {
+ fs.mkdirs(dstParent);
+ } else {
+ createAzCopyDirectory(dstParent);
+ }
+ } else {
+ createAzCopyFile(dstParent);
+ }
+ }
+
+ if (dstExist) {
+ if (!isDstFile) {
+ if (dstExplicit) {
+ fs.mkdirs(dst);
+ if(!dstParentExplicit) {
+ deleteBlobPath(fs, dstParent);
+ }
+ } else {
+ createAzCopyDirectory(dst);
+ }
+ if (subFileName != null) {
+ createAzCopyFile(new Path(dst, subFileName));
+ }
+ if (subDirName != null) {
+ if (isSubDirExplicit) {
+ fs.mkdirs(new Path(dst, subDirName));
+ if(!dstParentExplicit) {
+ deleteBlobPath(fs, dstParent);
+ }
+ if(!dstExplicit) {
+ deleteBlobPath(fs, dst);
+ }
+ } else {
+ createAzCopyDirectory(new Path(dst, subDirName));
+ }
+ }
+ } else {
+ createAzCopyFile(dst);
+ }
+ }
+ }
+
+ private void explicitImplicitCaseRenameAssert(final Boolean dstExist,
+ final Boolean shouldRenamePass,
+ final AzureBlobFileSystem fs,
+ final Path src,
+ final Path dst) throws IOException {
+ if (shouldRenamePass) {
+ Assert.assertTrue(fs.rename(src, dst));
+ if (dstExist) {
+ Assert.assertTrue(fs.getAbfsStore()
+ .getBlobProperty(new Path(dst, src.getName()),
+ getTestTracingContext(fs, true))
+ .getIsDirectory());
+ } else {
+ Assert.assertTrue(fs.getAbfsStore()
+ .getBlobProperty(dst, getTestTracingContext(fs, true))
+ .getIsDirectory());
+ }
+ } else {
+ Assert.assertFalse(fs.rename(src, dst));
+ Assert.assertTrue(fs.getAbfsStore()
+ .getListBlobs(src, null, null, getTestTracingContext(fs, true), null,
+ false)
+ .size() > 0);
+ if (dstExist) {
+ Assert.assertTrue(fs.getAbfsStore()
+ .getListBlobs(dst, null, null, getTestTracingContext(fs, true), null,
+ false)
+ .size() > 0);
+ }
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java
index b9498be89a1ffe..b4fb5d3f394ffc 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java
@@ -20,12 +20,27 @@
import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
+import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Assert;
+import org.junit.Assume;
import org.junit.Test;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.mockito.Mockito;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_DNS_PREFIX;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.WASB_DNS_PREFIX;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.mockito.Mockito.times;
/**
* Test FileStatus.
@@ -140,4 +155,265 @@ public void testLastModifiedTime() throws IOException {
assertTrue("lastModifiedTime should be before createEndTime",
createEndTime > lastModifiedTime);
}
+
+ @Test
+ public void testFileStatusOnFileWithImplicitParent() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ AzcopyHelper azcopyHelper = new AzcopyHelper(
+ getAccountName(),
+ getFileSystemName(),
+ getRawConfiguration(),
+ fs.getAbfsStore().getPrefixMode()
+ );
+
+ Path testPath = new Path("a/b.txt");
+ azcopyHelper.createFileUsingAzcopy(fs.makeQualified(testPath).toUri().getPath().substring(1));
+
+ assertTrue("Parent directory is implicit.",
+ BlobDirectoryStateHelper.isImplicitDirectory(testPath.getParent(), fs, getTestTracingContext(fs, true)));
+
+ // Assert getFileStatus Succeed on path
+ FileStatus fileStatus = fs.getFileStatus(testPath);
+ assertNotNull(fileStatus.getPath());
+ assertFalse(fileStatus.isDirectory());
+ assertNotEquals(0L, fileStatus.getLen());
+ }
+
+ @Test
+ public void testFileStatusOnFileWithExplicitParent() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path testPath = new Path("a/b.txt");
+ fs.create(testPath);
+
+ assertTrue("Parent directory is explicit.",
+ BlobDirectoryStateHelper.isExplicitDirectory(testPath.getParent(), fs, getTestTracingContext(fs, true)));
+
+ FileStatus fileStatus = fs.getFileStatus(testPath);
+ assertNotNull(fileStatus.getPath());
+ assertFalse(fileStatus.isDirectory());
+ }
+
+ @Test
+ public void testFileStatusOnImplicitDirWithImplicitParent() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ AzcopyHelper azcopyHelper = new AzcopyHelper(
+ getAccountName(),
+ getFileSystemName(),
+ getRawConfiguration(),
+ fs.getAbfsStore().getPrefixMode()
+ );
+
+ Path testPath = new Path("a/b");
+ azcopyHelper.createFolderUsingAzcopy(fs.makeQualified(testPath).toUri().getPath().substring(1));
+
+ assertTrue("Path is implicit.",
+ BlobDirectoryStateHelper.isImplicitDirectory(testPath, fs, getTestTracingContext(fs, true)));
+ assertTrue("Parent directory is implicit.",
+ BlobDirectoryStateHelper.isImplicitDirectory(testPath.getParent(), fs, getTestTracingContext(fs, true)));
+
+ // Assert that getFileStatus succeeds
+ FileStatus fileStatus = fs.getFileStatus(testPath);
+ assertNotNull(fileStatus.getPath());
+ assertTrue(fileStatus.isDirectory());
+ assertEquals(0L, fileStatus.getLen());
+ }
+
+ @Test
+ public void testFileStatusOnImplicitDirWithExplicitParent() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ AzcopyHelper azcopyHelper = new AzcopyHelper(
+ getAccountName(),
+ getFileSystemName(),
+ getRawConfiguration(),
+ fs.getAbfsStore().getPrefixMode()
+ );
+
+ Path testPath = new Path("a/b");
+ azcopyHelper.createFolderUsingAzcopy(fs.makeQualified(testPath).toUri().getPath().substring(1));
+ fs.mkdirs(testPath.getParent());
+
+ assertTrue("Path is implicit.",
+ BlobDirectoryStateHelper.isImplicitDirectory(testPath, fs, getTestTracingContext(fs, true)));
+ assertTrue("Parent directory is explicit.",
+ BlobDirectoryStateHelper.isExplicitDirectory(testPath.getParent(), fs, getTestTracingContext(fs, true)));
+
+ // Assert that getFileStatus succeeds
+ FileStatus fileStatus = fs.getFileStatus(testPath);
+ assertNotNull(fileStatus.getPath());
+ assertTrue(fileStatus.isDirectory());
+ assertEquals(0L, fileStatus.getLen());
+ }
+
+ @Test
+ public void testFileStatusOnExplicitDirWithExplicitParent() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path testPath = new Path("a/b");
+ fs.mkdirs(testPath);
+
+ assertTrue("Parent directory is explicit.",
+ BlobDirectoryStateHelper.isExplicitDirectory(testPath.getParent(), fs, getTestTracingContext(fs, true)));
+ assertTrue("Path is explicit.",
+ BlobDirectoryStateHelper.isExplicitDirectory(testPath, fs, getTestTracingContext(fs, true)));
+
+ // Assert that getFileStatus Succeeds
+ FileStatus fileStatus = fs.getFileStatus(testPath);
+ assertNotNull(fileStatus.getPath());
+ assertTrue(fileStatus.isDirectory());
+ assertEquals(0L, fileStatus.getLen());
+ }
+
+ @Test
+ public void testFileStatusOnNonExistingPathWithExplicitParent() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path testPath = new Path("a/b.txt");
+ fs.mkdirs(testPath.getParent());
+
+ assertTrue("Parent directory is explicit.",
+ BlobDirectoryStateHelper.isExplicitDirectory(testPath.getParent(), fs, getTestTracingContext(fs, true)));
+
+ // assert that getFileStatus fails
+ intercept(IOException.class,
+ () -> fs.getFileStatus(testPath));
+ }
+
+ @Test
+ public void testFileStatusOnNonExistingPathWithImplicitParent() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ AzcopyHelper azcopyHelper = new AzcopyHelper(
+ getAccountName(),
+ getFileSystemName(),
+ getRawConfiguration(),
+ fs.getAbfsStore().getPrefixMode()
+ );
+
+ Path testPath = new Path("a/b.txt");
+ azcopyHelper.createFolderUsingAzcopy(fs.makeQualified(
+ testPath.getParent()).toUri().getPath().substring(1));
+
+ assertTrue("Parent directory is implicit.",
+ BlobDirectoryStateHelper.isImplicitDirectory(testPath.getParent(), fs, getTestTracingContext(fs, true)));
+
+ // assert that getFileStatus Fails with IOException
+ intercept(IOException.class,
+ () -> fs.getFileStatus(testPath));
+ }
+
+ @Test
+ public void testFileStatusOnRoot() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path path = new Path("/");
+ fs.setWorkingDirectory(new Path("/"));
+
+ // Assert that getFileSus on root path succeed.
+ FileStatus fileStatus = fs.getFileStatus(path);
+ assertTrue(fileStatus.isDirectory());
+ assertTrue(fileStatus.getLen() == 0L);
+ }
+
+ @Test
+ public void testGetPathPropertyCalled() throws Exception {
+ Assume.assumeTrue(getFileSystem().getAbfsStore().getPrefixMode() == PrefixMode.BLOB);
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+
+ fs.create(new Path("/testGetPathProperty"));
+ FileStatus fileStatus = fs.getFileStatus(new Path("/testGetPathProperty"));
+
+ Assert.assertFalse(fileStatus.isDirectory());
+
+ Mockito.verify(store, times(2)).getPathProperty(Mockito.any(Path.class),
+ Mockito.any(TracingContext.class), Mockito.any(Boolean.class));
+ final AbfsConfiguration configuration= fs.getAbfsStore().getAbfsConfiguration();
+ final int listBlobAssertionTimes;
+ if (!configuration.shouldMkdirFallbackToDfs()
+ && !configuration.shouldReadFallbackToDfs()
+ && !configuration.shouldIngressFallbackToDfs()) {
+ listBlobAssertionTimes = 1;
+ } else{
+ listBlobAssertionTimes = 0;
+ }
+
+ Mockito.verify(store, times(listBlobAssertionTimes)).getListBlobs(Mockito.any(Path.class),
+ Mockito.nullable(String.class), Mockito.nullable(String.class),
+ Mockito.any(TracingContext.class), Mockito.any(Integer.class),
+ Mockito.any(Boolean.class));
+ }
+
+ @Test
+ public void testGetPathPropertyCalledImplicit() throws Exception {
+ Assume.assumeTrue(getFileSystem().getAbfsStore().getPrefixMode() == PrefixMode.BLOB);
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+
+ createAzCopyDirectory(new Path("/testImplicitDirectory"));
+ FileStatus fileStatus = fs.getFileStatus(new Path("/testImplicitDirectory"));
+
+ Assert.assertTrue(fileStatus.isDirectory());
+
+ final AbfsConfiguration configuration= fs.getAbfsStore().getAbfsConfiguration();
+ final int listBlobAssertionTimes;
+ if (!configuration.shouldMkdirFallbackToDfs()
+ && !configuration.shouldReadFallbackToDfs()
+ && !configuration.shouldIngressFallbackToDfs()) {
+ listBlobAssertionTimes = 1;
+ } else{
+ listBlobAssertionTimes = 0;
+ }
+
+ Mockito.verify(store, times(1)).getPathProperty(Mockito.any(Path.class),
+ Mockito.any(TracingContext.class), Mockito.any(Boolean.class));
+ Mockito.verify(store, times(listBlobAssertionTimes)).getListBlobs(Mockito.any(Path.class),
+ Mockito.nullable(String.class), Mockito.nullable(String.class),
+ Mockito.any(TracingContext.class), Mockito.any(Integer.class),
+ Mockito.any(Boolean.class));
+ }
+
+ @Test
+ public void testGetFileStatusReturnStatusWithPathWithSameUriGivenInConfig()
+ throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ String accountName = getAccountName();
+ Boolean isAccountNameInDfs = accountName.contains(ABFS_DNS_PREFIX);
+ String dnsAssertion;
+ if (isAccountNameInDfs) {
+ dnsAssertion = ABFS_DNS_PREFIX;
+ } else {
+ dnsAssertion = WASB_DNS_PREFIX;
+ }
+
+ final Path path = new Path("/testDir/file");
+ fs.create(path);
+ assertGetFileStatusPath(fs, accountName, dnsAssertion, path);
+
+ final Configuration configuration;
+ if (isAccountNameInDfs) {
+ configuration = new Configuration(getRawConfiguration());
+ configuration.set(FS_DEFAULT_NAME_KEY,
+ configuration.get(FS_DEFAULT_NAME_KEY)
+ .replace(ABFS_DNS_PREFIX, WASB_DNS_PREFIX));
+ dnsAssertion = WASB_DNS_PREFIX;
+
+ } else {
+ configuration = new Configuration(getRawConfiguration());
+ configuration.set(FS_DEFAULT_NAME_KEY,
+ configuration.get(FS_DEFAULT_NAME_KEY)
+ .replace(WASB_DNS_PREFIX, ABFS_DNS_PREFIX));
+ dnsAssertion = ABFS_DNS_PREFIX;
+ }
+ fs = (AzureBlobFileSystem) FileSystem.newInstance(configuration);
+ assertGetFileStatusPath(fs, accountName, dnsAssertion, path);
+ }
+
+ private void assertGetFileStatusPath(final AzureBlobFileSystem fs,
+ final String accountName,
+ final String dnsAssertion,
+ final Path path) throws IOException {
+ FileStatus fileStatus = fs.getFileStatus(new Path(
+ "abfs://" + fs.getAbfsClient().getFileSystem() + "@" + accountName
+ + path.toUri().getPath()));
+ Assertions.assertThat(fileStatus.getPath().toString())
+ .contains(dnsAssertion);
+ }
}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemLease.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemLease.java
index 07b8a6f2bb203d..0e7457e852e5b8 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemLease.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemLease.java
@@ -20,6 +20,10 @@
import java.io.IOException;
import java.util.concurrent.RejectedExecutionException;
+import org.apache.hadoop.fs.azurebfs.services.AbfsBlobLease;
+import org.apache.hadoop.fs.azurebfs.services.AbfsDfsLease;
+import org.apache.hadoop.fs.azurebfs.services.OperativeEndpoint;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
import org.junit.Assert;
import org.junit.Test;
@@ -37,6 +41,9 @@
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.LambdaTestUtils;
+import static org.apache.hadoop.fs.azurebfs.services.AbfsErrors.CONDITION_NOT_MET;
+import static org.apache.hadoop.fs.azurebfs.services.AbfsErrors.ERR_LEASE_EXPIRED_DFS;
+import static org.apache.hadoop.fs.azurebfs.services.AbfsErrors.ERR_NO_LEASE_ID_SPECIFIED_BLOB;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString;
@@ -123,7 +130,7 @@ public void testSubDir() throws Exception {
getCustomFileSystem(testFilePath.getParent().getParent(), 1);
fs.mkdirs(testFilePath.getParent().getParent());
- FSDataOutputStream out = fs.create(testFilePath);
+ FSDataOutputStream out = fs.create(testFilePath.getParent());
Assert.assertTrue("Output stream should have lease",
((AbfsOutputStream) out.getWrappedStream()).hasLease());
out.close();
@@ -137,10 +144,13 @@ public void testTwoCreate() throws Exception {
final Path testFilePath = new Path(path(methodName.getMethodName()), TEST_FILE);
final AzureBlobFileSystem fs = getCustomFileSystem(testFilePath.getParent(), 1);
fs.mkdirs(testFilePath.getParent());
-
+ PrefixMode prefixMode = getPrefixMode(fs);
try (FSDataOutputStream out = fs.create(testFilePath)) {
+ AbfsConfiguration abfsConfiguration = fs.getAbfsStore().getAbfsConfiguration();
LambdaTestUtils.intercept(IOException.class, isHNSEnabled ? ERR_PARALLEL_ACCESS_DETECTED
- : ERR_NO_LEASE_ID_SPECIFIED, () -> {
+ : !OperativeEndpoint.isIngressEnabledOnDFS(prefixMode, abfsConfiguration)
+ ? ERR_NO_LEASE_ID_SPECIFIED_BLOB
+ : ERR_NO_LEASE_ID_SPECIFIED, () -> {
try (FSDataOutputStream out2 = fs.create(testFilePath)) {
}
return "Expected second create on infinite lease dir to fail";
@@ -162,7 +172,23 @@ private void twoWriters(AzureBlobFileSystem fs, Path testFilePath, boolean expec
}
}
out.writeInt(1);
- out.hsync();
+ try {
+ out.hsync();
+ } catch (IOException e) {
+ // Etag mismatch leads to condition not met error for blob endpoint.
+ if (getPrefixMode(fs) == PrefixMode.BLOB) {
+ GenericTestUtils.assertExceptionContains(CONDITION_NOT_MET, e);
+ } else {
+ throw e;
+ }
+ }
+ } catch (IOException e) {
+ // Etag mismatch leads to condition not met error for blob endpoint.
+ if (getPrefixMode(fs) == PrefixMode.BLOB) {
+ GenericTestUtils.assertExceptionContains(CONDITION_NOT_MET, e);
+ } else {
+ throw e;
+ }
}
Assert.assertTrue("Store leases were not freed", fs.getAbfsStore().areLeasesFreed());
@@ -219,14 +245,17 @@ public void testWriteAfterBreakLease() throws Exception {
FSOperationType.BREAK_LEASE, false, 0));
fs.breakLease(testFilePath);
fs.registerListener(null);
-
- LambdaTestUtils.intercept(IOException.class, ERR_LEASE_EXPIRED, () -> {
+ PrefixMode prefixMode = getPrefixMode(fs);
+ AbfsConfiguration abfsConfiguration = fs.getAbfsStore().getAbfsConfiguration();
+ LambdaTestUtils.intercept(IOException.class, !OperativeEndpoint.isIngressEnabledOnDFS(prefixMode, abfsConfiguration)
+ ? ERR_LEASE_EXPIRED : ERR_LEASE_EXPIRED_DFS, () -> {
out.write(1);
out.hsync();
return "Expected exception on write after lease break but got " + out;
});
- LambdaTestUtils.intercept(IOException.class, ERR_LEASE_EXPIRED, () -> {
+ LambdaTestUtils.intercept(IOException.class, !OperativeEndpoint.isIngressEnabledOnDFS(prefixMode, abfsConfiguration)
+ ? ERR_LEASE_EXPIRED : ERR_LEASE_EXPIRED_DFS, () -> {
out.close();
return "Expected exception on close after lease break but got " + out;
});
@@ -252,8 +281,10 @@ public void testLeaseFreedAfterBreak() throws Exception {
out.write(0);
fs.breakLease(testFilePath);
-
- LambdaTestUtils.intercept(IOException.class, ERR_LEASE_EXPIRED, () -> {
+ PrefixMode prefixMode = getPrefixMode(fs);
+ AbfsConfiguration abfsConfiguration = fs.getAbfsStore().getAbfsConfiguration();
+ LambdaTestUtils.intercept(IOException.class, !OperativeEndpoint.isIngressEnabledOnDFS(prefixMode, abfsConfiguration)
+ ? ERR_LEASE_EXPIRED : ERR_LEASE_EXPIRED_DFS, () -> {
out.close();
return "Expected exception on close after lease break but got " + out;
});
@@ -318,8 +349,14 @@ public void testAcquireRetry() throws Exception {
FSOperationType.TEST_OP, true, 0);
tracingContext.setListener(listener);
- AbfsLease lease = new AbfsLease(fs.getAbfsClient(),
- testFilePath.toUri().getPath(), tracingContext);
+ AbfsLease lease;
+ if (getPrefixMode(fs) == PrefixMode.BLOB) {
+ lease = new AbfsBlobLease(fs.getAbfsClient(),
+ testFilePath.toUri().getPath(), null, tracingContext);
+ } else {
+ lease = new AbfsDfsLease(fs.getAbfsClient(),
+ testFilePath.toUri().getPath(), null, tracingContext);
+ }
Assert.assertNotNull("Did not successfully lease file", lease.getLeaseID());
listener.setOperation(FSOperationType.RELEASE_LEASE);
lease.free();
@@ -333,7 +370,20 @@ public void testAcquireRetry() throws Exception {
.doCallRealMethod().when(mockClient)
.acquireLease(anyString(), anyInt(), any(TracingContext.class));
- lease = new AbfsLease(mockClient, testFilePath.toUri().getPath(), 5, 1, tracingContext);
+ doThrow(new AbfsLease.LeaseException("failed to acquire 1"))
+ .doThrow(new AbfsLease.LeaseException("failed to acquire 2"))
+ .doCallRealMethod().when(mockClient)
+ .acquireBlobLease(anyString(), anyInt(), any(TracingContext.class));
+
+ if (getPrefixMode(fs) == PrefixMode.BLOB) {
+ lease = new AbfsBlobLease(mockClient, testFilePath.toUri().getPath(), 5,
+ 1, null,
+ tracingContext);
+ } else {
+ lease = new AbfsDfsLease(mockClient, testFilePath.toUri().getPath(), 5, 1,
+ null,
+ tracingContext);
+ }
Assert.assertNotNull("Acquire lease should have retried", lease.getLeaseID());
lease.free();
Assert.assertEquals("Unexpected acquire retry count", 2, lease.getAcquireRetryCount());
@@ -341,9 +391,18 @@ public void testAcquireRetry() throws Exception {
doThrow(new AbfsLease.LeaseException("failed to acquire")).when(mockClient)
.acquireLease(anyString(), anyInt(), any(TracingContext.class));
+ doThrow(new AbfsLease.LeaseException("failed to acquire")).when(mockClient)
+ .acquireBlobLease(anyString(), anyInt(), any(TracingContext.class));
+
LambdaTestUtils.intercept(AzureBlobFileSystemException.class, () -> {
- new AbfsLease(mockClient, testFilePath.toUri().getPath(), 5, 1,
- tracingContext);
+ if (getPrefixMode(fs) == PrefixMode.BLOB) {
+ new AbfsBlobLease(mockClient, testFilePath.toUri().getPath(), 5, 1,
+ null,
+ tracingContext);
+ } else {
+ new AbfsDfsLease(mockClient, testFilePath.toUri().getPath(), 5, 1, null,
+ tracingContext);
+ }
});
}
}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java
index dc9ef9bffb9452..4cae9a4ac5f565 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java
@@ -20,6 +20,7 @@
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.net.SocketTimeoutException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
@@ -27,6 +28,16 @@
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants;
+import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
+import org.apache.hadoop.fs.azurebfs.services.AbfsClientTestUtil;
+import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperationTestUtil;
+import org.apache.hadoop.fs.azurebfs.services.BlobList;
+import org.apache.hadoop.fs.azurebfs.services.BlobProperty;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
+import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
+import org.assertj.core.api.Assertions;
+import org.junit.Assume;
import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
@@ -36,16 +47,29 @@
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azurebfs.constants.FSOperationType;
+import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderFormat;
import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderValidator;
import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.mockito.Mockito;
+import org.mockito.stubbing.Stubber;
+
+import static java.net.HttpURLConnection.HTTP_OK;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_LIST_MAX_RESULTS;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_DNS_PREFIX;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.WASB_DNS_PREFIX;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_TIMEOUT_JDK_MESSAGE;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertMkdirs;
import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathExists;
import static org.apache.hadoop.fs.contract.ContractTestUtils.rename;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.nullable;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.when;
/**
* Test listStatus operation.
@@ -60,7 +84,7 @@ public ITestAzureBlobFileSystemListStatus() throws Exception {
@Test
public void testListPath() throws Exception {
- Configuration config = new Configuration(this.getRawConfiguration());
+ Configuration config = Mockito.spy(this.getRawConfiguration());
config.set(AZURE_LIST_MAX_RESULTS, "5000");
final AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem
.newInstance(getFileSystem().getUri(), config);
@@ -92,6 +116,62 @@ public Void call() throws Exception {
assertEquals(TEST_FILES_NUMBER, files.length /* user directory */);
}
+ /**
+ * Test to verify that each paginated call to ListBlobs uses a new tracing context.
+ * @throws Exception
+ */
+ @Test
+ public void testListPathTracingContext() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ Assume.assumeTrue("To work on only on non-HNS Blob endpoint",
+ fs.getAbfsStore().getAbfsConfiguration().getPrefixMode()
+ == PrefixMode.BLOB);
+ TracingHeaderValidator validator = new TracingHeaderValidator(getConfiguration().getClientCorrelationId(),
+ fs.getFileSystemId(), FSOperationType.LISTSTATUS, true, 0);
+ validator.setDisableValidation(true);
+ fs.registerListener(validator);
+
+ final AzureBlobFileSystem spiedFs = Mockito.spy(fs);
+ final AzureBlobFileSystemStore spiedStore = Mockito.spy(fs.getAbfsStore());
+ final AbfsClient spiedClient = Mockito.spy(fs.getAbfsClient());
+ final TracingContext spiedTracingContext = Mockito.spy(
+ new TracingContext(
+ fs.getClientCorrelationId(), fs.getFileSystemId(),
+ FSOperationType.LISTSTATUS, true, TracingHeaderFormat.ALL_ID_FORMAT,validator));
+
+ Mockito.doReturn(spiedStore).when(spiedFs).getAbfsStore();
+ spiedStore.setClient(spiedClient);
+ spiedFs.setWorkingDirectory(new Path("/"));
+
+ AbfsClientTestUtil.setMockAbfsRestOperationForListBlobOperation(spiedClient,
+ (httpOperation) -> {
+ BlobProperty blob = new BlobProperty();
+ blob.setPath(new Path("/abc.txt"));
+ blob.setName("abc.txt");
+ BlobList blobListWithNextMarker = new BlobList();
+ AbfsClientTestUtil.populateBlobListHelper(blobListWithNextMarker, blob, "nextMarker");
+ BlobList blobListWithoutNextMarker = new BlobList();
+ AbfsClientTestUtil.populateBlobListHelper(blobListWithNextMarker, blob, AbfsHttpConstants.EMPTY_STRING);
+ when(httpOperation.getBlobList()).thenReturn(blobListWithNextMarker)
+ .thenReturn(blobListWithoutNextMarker);
+
+ Stubber stubber = Mockito.doThrow(
+ new SocketTimeoutException(CONNECTION_TIMEOUT_JDK_MESSAGE));
+ stubber.doNothing().when(httpOperation).processResponse(
+ nullable(byte[].class), nullable(int.class), nullable(int.class));
+
+ when(httpOperation.getStatusCode()).thenReturn(-1).thenReturn(HTTP_OK);
+ return httpOperation;
+ });
+
+ List fileStatuses = new ArrayList<>();
+ spiedStore.listStatus(new Path("/"), "", fileStatuses, true, null, spiedTracingContext);
+
+ // Assert that there were 2 paginated ListBlob calls made and new tracing context were created.
+ Mockito.verify(spiedClient, times(2)).getListBlobs(any(), any(), any(), any(), any());
+ Mockito.verify(spiedTracingContext, times(0)).constructHeader(any(), any());
+ }
+
/**
* Creates a file, verifies that listStatus returns it,
* even while the file is still open for writing.
@@ -247,4 +327,328 @@ public void testRenameTrailingPeriodFile() throws IOException {
assertTrue("Attempt to create file that ended with a dot should"
+ " throw IllegalArgumentException", exceptionThrown);
}
+
+ @Test
+ public void testListStatusImplicitExplicitChildren() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.setWorkingDirectory(new Path("/"));
+ Path root = new Path("/");
+ AzcopyHelper azcopyHelper = new AzcopyHelper(
+ getAccountName(),
+ getFileSystemName(),
+ getRawConfiguration(),
+ fs.getAbfsStore().getPrefixMode()
+ );
+
+ // Create an implicit directory under root
+ Path dir1 = new Path("a");
+ azcopyHelper.createFolderUsingAzcopy(fs.makeQualified(dir1).toUri().getPath().substring(1));
+ assertTrue("Path is implicit.",
+ BlobDirectoryStateHelper.isImplicitDirectory(dir1, fs, getTestTracingContext(fs, true)));
+
+ // Assert that implicit directory is returned
+ FileStatus[] fileStatuses = fs.listStatus(root);
+ Assertions.assertThat(fileStatuses.length).isEqualTo(1);
+ assertImplicitDirectoryFileStatus(fileStatuses[0], fs.makeQualified(dir1));
+
+ // Create a marker blob for the directory.
+ fs.mkdirs(dir1);
+
+ // Assert that only one entry of explicit directory is returned
+ fileStatuses = fs.listStatus(root);
+ Assertions.assertThat(fileStatuses.length).isEqualTo(1);
+ assertExplicitDirectoryFileStatus(fileStatuses[0], fs.makeQualified(dir1));
+
+ // Create a file under root
+ Path file1 = new Path("b");
+ fs.create(file1);
+
+ // Assert that two entries are returned in alphabetic order.
+ fileStatuses = fs.listStatus(root);
+ Assertions.assertThat(fileStatuses.length).isEqualTo(2);
+ assertExplicitDirectoryFileStatus(fileStatuses[0], fs.makeQualified(dir1));
+ assertFileFileStatus(fileStatuses[1], fs.makeQualified(file1));
+
+ // Create another implicit directory under root.
+ Path dir2 = new Path("c");
+ azcopyHelper.createFolderUsingAzcopy(fs.makeQualified(dir2).toUri().getPath().substring(1));
+ assertTrue("Path is implicit.",
+ BlobDirectoryStateHelper.isImplicitDirectory(dir2, fs, getTestTracingContext(fs, true)));
+
+ // Assert that three entries are returned in alphabetic order.
+ fileStatuses = fs.listStatus(root);
+ Assertions.assertThat(fileStatuses.length).isEqualTo(3);
+ assertExplicitDirectoryFileStatus(fileStatuses[0], fs.makeQualified(dir1));
+ assertFileFileStatus(fileStatuses[1], fs.makeQualified(file1));
+ assertImplicitDirectoryFileStatus(fileStatuses[2], fs.makeQualified(dir2));
+ }
+
+ @Test
+ public void testListStatusOnNonExistingPath() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path testPath = new Path("a/b");
+
+ intercept(FileNotFoundException.class,
+ () -> fs.listFiles(testPath, false).next());
+ }
+
+ @Test
+ public void testListStatusOnImplicitDirectory() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ AzcopyHelper azcopyHelper = new AzcopyHelper(
+ getAccountName(),
+ getFileSystemName(),
+ getRawConfiguration(),
+ fs.getAbfsStore().getPrefixMode()
+ );
+
+ // Create an implicit directory with another implicit directory inside
+ Path testPath = new Path("testDir");
+ Path childPath = new Path("testDir/azcopy");
+ azcopyHelper.createFolderUsingAzcopy(
+ fs.makeQualified(testPath).toUri().getPath().substring(1));
+ assertTrue("Path is implicit.",
+ BlobDirectoryStateHelper.isImplicitDirectory(testPath, fs, getTestTracingContext(fs, true)));
+
+ // Assert that one entry is returned as implicit child.
+ FileStatus[] fileStatuses = fs.listStatus(testPath);
+ Assertions.assertThat(fileStatuses.length).isEqualTo(1);
+ assertImplicitDirectoryFileStatus(fileStatuses[0], fs.makeQualified(childPath));
+ }
+
+ @Test
+ public void testListStatusOnExplicitDirectory() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ AzcopyHelper azcopyHelper = new AzcopyHelper(
+ getAccountName(),
+ getFileSystemName(),
+ getRawConfiguration(),
+ fs.getAbfsStore().getPrefixMode()
+ );
+
+ // Create an explicit directory with all kind of children.
+ Path testPath = new Path("testDir");
+ Path explicitChild = new Path ("testDir/a/subdir");
+ Path fileChild = new Path ("testDir/b");
+ Path implicitChild = new Path ("testDir/c");
+ fs.mkdirs(explicitChild);
+ fs.create(fileChild);
+ azcopyHelper.createFolderUsingAzcopy(
+ fs.makeQualified(implicitChild).toUri().getPath().substring(1));
+
+ assertTrue("Test path is explicit",
+ BlobDirectoryStateHelper.isExplicitDirectory(testPath, fs, getTestTracingContext(fs, true)));
+ assertTrue("explicitChild Path is explicit",
+ BlobDirectoryStateHelper.isExplicitDirectory(explicitChild, fs, getTestTracingContext(fs, true)));
+ assertTrue("implicitChild Path is implicit.",
+ BlobDirectoryStateHelper.isImplicitDirectory(implicitChild, fs, getTestTracingContext(fs, true)));
+
+ // Assert that three entry is returned.
+ FileStatus[] fileStatuses = fs.listStatus(testPath);
+ Assertions.assertThat(fileStatuses.length).isEqualTo(3);
+ assertExplicitDirectoryFileStatus(fileStatuses[0], fs.makeQualified(explicitChild.getParent()));
+ assertFileFileStatus(fileStatuses[1], fs.makeQualified(fileChild));
+ assertImplicitDirectoryFileStatus(fileStatuses[2], fs.makeQualified(implicitChild));
+ }
+
+ @Test
+ public void testListStatusImplicitExplicitWithDotInFolderName() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ AzcopyHelper azcopyHelper = new AzcopyHelper(
+ getAccountName(),
+ getFileSystemName(),
+ getRawConfiguration(),
+ fs.getAbfsStore().getPrefixMode()
+ );
+
+ // Create two implicit folder with same prefix one having dot.
+ Path testPath = new Path("Try1/DirA");
+ Path implicitChild1 = new Path ("Try1/DirA/DirB/file.txt");
+ Path implicitChild2 = new Path ("Try1/DirA/DirB.bak/file.txt");
+ Path explicitChild = new Path ("Try1/DirA/DirB");
+
+ azcopyHelper.createFolderUsingAzcopy(
+ fs.makeQualified(implicitChild1).toUri().getPath().substring(1));
+ azcopyHelper.createFolderUsingAzcopy(
+ fs.makeQualified(implicitChild2).toUri().getPath().substring(1));
+ fs.mkdirs(explicitChild);
+
+ assertTrue("Test path is explicit",
+ BlobDirectoryStateHelper.isExplicitDirectory(testPath, fs, getTestTracingContext(fs, true)));
+ assertTrue("explicitChild Path is explicit",
+ BlobDirectoryStateHelper.isExplicitDirectory(explicitChild, fs, getTestTracingContext(fs, true)));
+ assertTrue("implicitChild2 Path is implicit.",
+ BlobDirectoryStateHelper.isImplicitDirectory(implicitChild2, fs, getTestTracingContext(fs, true)));
+
+ // Assert that only 2 entry is returned.
+ FileStatus[] fileStatuses = fs.listStatus(testPath);
+ Assertions.assertThat(fileStatuses.length).isEqualTo(2);
+ assertExplicitDirectoryFileStatus(fileStatuses[0], fs.makeQualified(explicitChild));
+ assertImplicitDirectoryFileStatus(fileStatuses[1], fs.makeQualified(implicitChild2.getParent()));
+ }
+
+ @Test
+ public void testListStatusOnEmptyDirectory() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path testPath = new Path("testPath");
+ fs.mkdirs(testPath);
+ FileStatus[] fileStatuses = fs.listStatus(testPath);
+ Assertions.assertThat(fileStatuses.length).isEqualTo(0);
+ }
+
+ @Test
+ public void testListStatusUsesGfs() throws Exception {
+ Assume.assumeTrue(getFileSystem().getAbfsStore().getPrefixMode() == PrefixMode.BLOB);
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ AbfsClient client = Mockito.spy(store.getClient());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ Mockito.doReturn(client).when(store).getClient();
+ store.setClient(client);
+
+ intercept(FileNotFoundException.class, () ->
+ fs.listStatus(new Path("a/b")));
+
+ Mockito.verify(store, times(1)).getPathProperty(Mockito.any(Path.class),
+ Mockito.any(TracingContext.class), Mockito.any(boolean.class));
+
+ // getListBlobs from within store should not be called - as this is invoked from getFileStatus
+ Mockito.verify(store, times(0)).getListBlobs(Mockito.any(Path.class),
+ Mockito.any(String.class), Mockito.any(String.class), Mockito.any(TracingContext.class),
+ Mockito.any(Integer.class), Mockito.any(Boolean.class));
+
+ // getListBlobs from client should be called once - the call for listStatus that would fail
+ // as this blob is actually non-existent
+ Mockito.verify(client, times(1)).getListBlobs(Mockito.nullable(String.class),
+ Mockito.nullable(String.class), Mockito.nullable(String.class),
+ Mockito.any(Integer.class), Mockito.any(TracingContext.class));
+ }
+
+ private void assertFileFileStatus(final FileStatus fileStatus,
+ final Path qualifiedPath) {
+ Assertions.assertThat(fileStatus.getPath()).isEqualTo(qualifiedPath);
+ Assertions.assertThat(fileStatus.isDirectory()).isEqualTo(false);
+ Assertions.assertThat(fileStatus.isFile()).isEqualTo(true);
+ Assertions.assertThat(fileStatus.getModificationTime()).isNotEqualTo(0);
+ }
+
+ private void assertImplicitDirectoryFileStatus(final FileStatus fileStatus,
+ final Path qualifiedPath) {
+ assertDirectoryFileStatus(fileStatus, qualifiedPath);
+ Assertions.assertThat(fileStatus.getModificationTime()).isEqualTo(0);
+ }
+
+ private void assertExplicitDirectoryFileStatus(final FileStatus fileStatus,
+ final Path qualifiedPath) {
+ assertDirectoryFileStatus(fileStatus, qualifiedPath);
+ Assertions.assertThat(fileStatus.getModificationTime()).isNotEqualTo(0);
+ }
+
+ private void assertDirectoryFileStatus(final FileStatus fileStatus,
+ final Path qualifiedPath) {
+ Assertions.assertThat(fileStatus.getPath()).isEqualTo(qualifiedPath);
+ Assertions.assertThat(fileStatus.isDirectory()).isEqualTo(true);
+ Assertions.assertThat(fileStatus.isFile()).isEqualTo(false);
+ Assertions.assertThat(fileStatus.getLen()).isEqualTo(0);
+ }
+
+ @Test
+ public void testListStatusNotTriesToRenameResumeForNonAtomicDir()
+ throws Exception {
+ Assume.assumeTrue(getPrefixMode(getFileSystem()) == PrefixMode.BLOB);
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ Mockito.doReturn(new FileStatus[1])
+ .when(store)
+ .listStatus(Mockito.any(Path.class), Mockito.any(
+ TracingContext.class));
+ fs.listStatus(new Path("/testDir/"));
+ Mockito.verify(store, Mockito.times(0))
+ .getRenamePendingFileStatus(Mockito.any(FileStatus[].class));
+ }
+
+ @Test
+ public void testListStatusTriesToRenameResumeForAtomicDir() throws Exception {
+ Assume.assumeTrue(getPrefixMode(getFileSystem()) == PrefixMode.BLOB);
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ Mockito.doReturn(new FileStatus[0])
+ .when(store)
+ .listStatus(Mockito.any(Path.class), Mockito.any(
+ TracingContext.class));
+ fs.listStatus(new Path("/hbase/"));
+ Mockito.verify(store, Mockito.times(1))
+ .getRenamePendingFileStatus(Mockito.any(FileStatus[].class));
+ }
+
+ @Test
+ public void testListStatusTriesToRenameResumeForAbsoluteAtomicDir()
+ throws Exception {
+ Assume.assumeTrue(getPrefixMode(getFileSystem()) == PrefixMode.BLOB);
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ Mockito.doReturn(new FileStatus[0])
+ .when(store)
+ .listStatus(Mockito.any(Path.class), Mockito.any(
+ TracingContext.class));
+ fs.listStatus(new Path("/hbase"));
+ Mockito.verify(store, Mockito.times(1))
+ .getRenamePendingFileStatus(Mockito.any(FileStatus[].class));
+ }
+
+ @Test
+ public void testListStatusReturnStatusWithPathWithSameUriGivenInConfig()
+ throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ String accountName = getAccountName();
+ Boolean isAccountNameInDfs = accountName.contains(ABFS_DNS_PREFIX);
+ String dnsAssertion;
+ if (isAccountNameInDfs) {
+ dnsAssertion = ABFS_DNS_PREFIX;
+ } else {
+ dnsAssertion = WASB_DNS_PREFIX;
+ }
+
+ final Path path = new Path("/testDir/file");
+ fs.create(path);
+ assertListStatusPath(fs, accountName, dnsAssertion, path);
+
+ final Configuration configuration;
+ if (isAccountNameInDfs) {
+ configuration = new Configuration(getRawConfiguration());
+ configuration.set(FS_DEFAULT_NAME_KEY,
+ configuration.get(FS_DEFAULT_NAME_KEY)
+ .replace(ABFS_DNS_PREFIX, WASB_DNS_PREFIX));
+ dnsAssertion = WASB_DNS_PREFIX;
+
+ } else {
+ configuration = new Configuration(getRawConfiguration());
+ configuration.set(FS_DEFAULT_NAME_KEY,
+ configuration.get(FS_DEFAULT_NAME_KEY)
+ .replace(WASB_DNS_PREFIX, ABFS_DNS_PREFIX));
+ dnsAssertion = ABFS_DNS_PREFIX;
+ }
+ fs = (AzureBlobFileSystem) FileSystem.newInstance(configuration);
+ assertListStatusPath(fs, accountName, dnsAssertion, path);
+ }
+
+ private void assertListStatusPath(final AzureBlobFileSystem fs,
+ final String accountName,
+ final String dnsAssertion,
+ final Path path) throws IOException {
+ FileStatus[] fileStatuses = fs.listStatus(new Path(
+ "abfs://" + fs.getAbfsClient().getFileSystem() + "@" + accountName
+ + path.toUri().getPath()));
+ Assertions.assertThat(fileStatuses[0].getPath().toString())
+ .contains(dnsAssertion);
+
+ fileStatuses = fs.listStatus(new Path(
+ "abfs://" + fs.getAbfsClient().getFileSystem() + "@" + accountName
+ + path.getParent().toUri().getPath()));
+ Assertions.assertThat(fileStatuses[0].getPath().toString())
+ .contains(dnsAssertion);
+ }
}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java
index a9a059e7cda560..688e9b06a82c34 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java
@@ -20,6 +20,12 @@
import java.util.UUID;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
+import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
+import org.apache.hadoop.fs.azurebfs.services.OperativeEndpoint;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
+import org.apache.hadoop.fs.azurebfs.services.ITestAbfsClient;
+import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
import org.junit.Assume;
import org.junit.Test;
@@ -27,9 +33,11 @@
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.mockito.Mockito;
import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.CONNECTIONS_MADE;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_MKDIR_OVERWRITE;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_MKDIRS_FALLBACK_TO_DFS;
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_FS_AZURE_ENABLE_MKDIR_OVERWRITE;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertMkdirs;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@@ -122,8 +130,12 @@ public void testCreateDirOverwrite(boolean enableConditionalCreateOverwrite)
// Case 1: Dir does not pre-exist
fs.mkdirs(dirPath);
- // One request to server
- mkdirRequestCount++;
+ // One request to server for dfs and 2 for blob because child calls mkdir for parent.
+ if (!OperativeEndpoint.isMkdirEnabledOnDFS(getAbfsStore(fs).getAbfsConfiguration())) {
+ mkdirRequestCount += 2;
+ } else {
+ mkdirRequestCount++;
+ }
assertAbfsStatistics(
CONNECTIONS_MADE,
@@ -134,12 +146,88 @@ public void testCreateDirOverwrite(boolean enableConditionalCreateOverwrite)
// Mkdir on existing Dir path will not lead to failure
fs.mkdirs(dirPath);
- // One request to server
- mkdirRequestCount++;
+ // One request to server for dfs and 3 for blob because child calls mkdir for parent.
+ if (!OperativeEndpoint.isMkdirEnabledOnDFS(getAbfsStore(fs).getAbfsConfiguration())) {
+ mkdirRequestCount += 3;
+ } else {
+ mkdirRequestCount++;
+ }
assertAbfsStatistics(
CONNECTIONS_MADE,
totalConnectionMadeBeforeTest + mkdirRequestCount,
fs.getInstrumentationMap());
}
+
+ @Test
+ public void testVerifyGetBlobProperty() throws Exception {
+ Assume.assumeTrue(getFileSystem().getAbfsStore().getPrefixMode() == PrefixMode.BLOB);
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ AbfsClient client = store.getClient();
+ AbfsClient testClient = Mockito.spy(ITestAbfsClient.createTestClientFromCurrentContext(
+ client,
+ fs.getAbfsStore().getAbfsConfiguration()));
+ store.setClient(testClient);
+
+ createAzCopyDirectory(new Path("/src"));
+ intercept(AbfsRestOperationException.class, () -> {
+ store.getBlobProperty(new Path("/src"), getTestTracingContext(fs, true));
+ });
+ fs.mkdirs(new Path("/src/dir"));
+ Mockito.verify(testClient, Mockito.times(0)).getPathStatus(Mockito.any(String.class),
+ Mockito.anyBoolean(), Mockito.any(TracingContext.class));
+ Mockito.verify(testClient, Mockito.atLeast(1)).getBlobProperty(Mockito.any(Path.class),
+ Mockito.any(TracingContext.class));
+
+ }
+
+ @Test
+ public void testGetPathPropertyCalledOnMkdirBlobEndpoint() throws Exception {
+ Assume.assumeTrue(getFileSystem().getAbfsStore().getPrefixMode() == PrefixMode.BLOB);
+ AbfsConfiguration abfsConfig = new AbfsConfiguration(getRawConfiguration(), getAccountName());
+ abfsConfig.setBoolean(FS_AZURE_MKDIRS_FALLBACK_TO_DFS, false);
+ AzureBlobFileSystem fs = Mockito.spy((AzureBlobFileSystem) FileSystem.newInstance(abfsConfig.getRawConfiguration()));
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+
+ fs.mkdirs(new Path("/testPath"));
+ Mockito.verify(store, Mockito.times(1)).tryGetPathProperty(Mockito.any(Path.class),
+ Mockito.any(TracingContext.class), Mockito.any(Boolean.class));
+ Mockito.verify(store, Mockito.times(1)).getPathProperty(Mockito.any(Path.class),
+ Mockito.any(TracingContext.class), Mockito.any(Boolean.class));
+
+ }
+
+ @Test
+ public void testGetPathPropertyCalledOnMkdirFallbackDFS() throws Exception {
+ Assume.assumeTrue(getFileSystem().getAbfsStore().getPrefixMode() == PrefixMode.BLOB);
+ AbfsConfiguration abfsConfig = new AbfsConfiguration(getRawConfiguration(), getAccountName());
+ abfsConfig.setBoolean(FS_AZURE_MKDIRS_FALLBACK_TO_DFS, true);
+ AzureBlobFileSystem fs = Mockito.spy((AzureBlobFileSystem) FileSystem.newInstance(abfsConfig.getRawConfiguration()));
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+
+
+ fs.mkdirs(new Path("/testPath"));
+ Mockito.verify(store, Mockito.times(0)).tryGetPathProperty(Mockito.any(Path.class),
+ Mockito.any(TracingContext.class), Mockito.any(Boolean.class));
+ Mockito.verify(store, Mockito.times(0)).getPathProperty(Mockito.any(Path.class),
+ Mockito.any(TracingContext.class), Mockito.any(Boolean.class));
+
+ }
+
+ @Test
+ public void testMkdirWithExistingFilenameBlobEndpoint() throws Exception {
+ Assume.assumeTrue(getFileSystem().getAbfsStore().getPrefixMode() == PrefixMode.BLOB);
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+
+ fs.create(new Path("/testFilePath"));
+ intercept(FileAlreadyExistsException.class, () -> fs.mkdirs(new Path("/testFilePath")));
+ intercept(FileAlreadyExistsException.class, () -> fs.mkdirs(new Path("/testFilePath/newDir")));
+ }
}
+
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java
index 0a9713cffa94b5..6b6e5e7f23581a 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java
@@ -18,14 +18,21 @@
package org.apache.hadoop.fs.azurebfs;
import java.io.EOFException;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Random;
import java.util.concurrent.Callable;
import java.util.UUID;
+import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
+import org.apache.hadoop.fs.azurebfs.services.OperativeEndpoint;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
+import org.apache.hadoop.fs.azurebfs.services.ITestAbfsClient;
+import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
import org.junit.Assume;
import org.junit.Ignore;
import org.junit.Test;
+import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -574,6 +581,49 @@ public void testAlwaysReadBufferSizeConfig(boolean alwaysReadBufferSizeConfigVal
assertAbfsStatistics(BYTES_RECEIVED, dateSizeReadStatAtStart + newDataSizeRead, fs.getInstrumentationMap());
}
+ @Test
+ public void testReadBlob() throws IOException {
+ Assume.assumeTrue(PrefixMode.BLOB == getFileSystem().getAbfsStore().getPrefixMode());
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ AbfsClient client = store.getClient();
+ AbfsClient mockClient = Mockito.spy(ITestAbfsClient.createTestClientFromCurrentContext(
+ client,
+ fs.getAbfsStore().getAbfsConfiguration()
+ ));
+ store.setClient(mockClient);
+ Mockito.doReturn(mockClient).when(store).getClient();
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+
+ Path testPath = new Path("/testReadFile");
+ fs.create(testPath);
+ FSDataInputStream in = fs.open(testPath);
+ if (!OperativeEndpoint.isReadEnabledOnDFS(store.getAbfsConfiguration())) {
+ Mockito.verify(mockClient, Mockito.atLeast(1)).getBlobProperty(
+ Mockito.any(Path.class), Mockito.any(TracingContext.class));
+ } else {
+ Mockito.verify(mockClient, Mockito.times(2)).getPathStatus(
+ Mockito.any(String.class), Mockito.anyBoolean(), Mockito.any(TracingContext.class));
+ }
+ }
+
+ @Test
+ public void testInvalidImplicitDirRead() throws Exception {
+ AzureBlobFileSystem fs = (AzureBlobFileSystem) getFileSystem();
+ AzcopyHelper azcopyhelper = new AzcopyHelper(getAccountName(),
+ getFileSystemName(),
+ getRawConfiguration(),
+ fs.getAbfsStore().getPrefixMode());
+ String fullPath = "/implicitDirPath/testFile";
+ String path = "/implicitDirPath";
+ azcopyhelper.createFolderUsingAzcopy(
+ fs.makeQualified(new Path(fullPath)).toUri().getPath().substring(1)
+ );
+
+ intercept(FileNotFoundException.class, () ->
+ fs.open(new Path(path)));
+
+ }
private long sequentialRead(String version,
Path testPath,
FileSystem fs,
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java
index 48c3a9d9665e33..fd415538f4a3f5 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java
@@ -18,23 +18,76 @@
package org.apache.hadoop.fs.azurebfs;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.SocketException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.Random;
import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+
+import org.assertj.core.api.Assertions;
import org.junit.Assert;
+import org.junit.Assume;
import org.junit.Test;
+import org.mockito.Mockito;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.constants.FSOperationType;
+import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
+import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode;
+import org.apache.hadoop.fs.azurebfs.services.AbfsBlobLease;
+import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
+import org.apache.hadoop.fs.azurebfs.services.AbfsClientTestUtil;
+import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOpTestUtil;
+import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation;
+import org.apache.hadoop.fs.azurebfs.services.AbfsInputStream;
+import org.apache.hadoop.fs.azurebfs.services.AbfsLease;
+import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation;
+import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperationTestUtil;
+import org.apache.hadoop.fs.azurebfs.services.ListBlobProducer;
+import org.apache.hadoop.fs.azurebfs.services.ListBlobQueue;
+import org.apache.hadoop.fs.azurebfs.services.RenameAtomicityUtils;
+import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
+import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderValidator;
+import org.apache.hadoop.fs.impl.OpenFileParameters;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_LEASE_CREATE_NON_RECURSIVE;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_MKDIRS_FALLBACK_TO_DFS;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_REDIRECT_RENAME;
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.TRUE;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_INGRESS_FALLBACK_TO_DFS;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_MB;
+import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_LEASE_ID;
+import static org.apache.hadoop.fs.azurebfs.services.RenameAtomicityUtils.SUFFIX;
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.COPY_STATUS_ABORTED;
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.COPY_STATUS_FAILED;
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.COPY_STATUS_PENDING;
+import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.COPY_BLOB_ABORTED;
+import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.COPY_BLOB_FAILED;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsFile;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertMkdirs;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathDoesNotExist;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertRenameOutcome;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
/**
* Test rename operation.
@@ -87,6 +140,28 @@ public void testRenameFileUnderDir() throws Exception {
filename, status.getPath().getName());
}
+ @Test
+ public void testRenameFileUnderDirRedirection() throws Exception {
+ Configuration configuration = Mockito.spy(getRawConfiguration());
+
+ // Set redirect to wasb rename true and assert rename.
+ configuration.setBoolean(FS_AZURE_REDIRECT_RENAME, true);
+ AzureBlobFileSystem fs1 = (AzureBlobFileSystem) FileSystem.newInstance(configuration);
+ Path sourceDir = makeQualified(new Path("/testSrc"));
+ assertMkdirs(fs1, sourceDir);
+ String filename = "file1";
+ Path file1 = new Path(sourceDir, filename);
+ touch(file1);
+
+ Path destDir = makeQualified(new Path("/testDst"));
+ assertRenameOutcome(fs1, sourceDir, destDir, true);
+ FileStatus[] fileStatus = fs1.listStatus(destDir);
+ assertNotNull("Null file status", fileStatus);
+ FileStatus status = fileStatus[0];
+ assertEquals("Wrong filename in " + status,
+ filename, status.getPath().getName());
+ }
+
@Test
public void testRenameDirectory() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
@@ -160,4 +235,2279 @@ public void testPosixRenameDirectory() throws Exception {
assertFalse(fs.exists(new Path("testDir2/test1/test2/test3")));
}
+ @Test
+ public void testRenameToRoot() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("/src1/src2"));
+ Assert.assertTrue(fs.rename(new Path("/src1/src2"), new Path("/")));
+ Assert.assertTrue(fs.exists(new Path("/src2")));
+ }
+
+ @Test
+ public void testRenameNotFoundBlobToEmptyRoot() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ Assert.assertFalse(fs.rename(new Path("/file"), new Path("/")));
+ }
+
+ @Test(expected = IOException.class)
+ public void testRenameBlobToDstWithColonInPath() throws Exception{
+ AzureBlobFileSystem fs = getFileSystem();
+ assumeNonHnsAccountBlobEndpoint(fs);
+ fs.create(new Path("/src"));
+ fs.rename(new Path("/src"), new Path("/dst:file"));
+ }
+
+ @Test
+ public void testRenameBlobInSameDirectoryWithNoMarker() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ assumeNonHnsAccountBlobEndpoint(fs);
+ fs.create(new Path("/srcDir/dir/file"));
+ fs.getAbfsStore().getClient().deleteBlobPath(new Path("/srcDir/dir"), null,
+ getTestTracingContext(fs, true));
+ Assert.assertTrue(fs.rename(new Path("/srcDir/dir"), new Path("/srcDir")));
+ }
+
+ /**
+ *
+ * Test to check behaviour of rename API if the destination directory is already
+ * there. The HNS call and the one for Blob endpoint should have same behaviour.
+ *
+ * /testDir2/test1/test2/test3 contains (/file)
+ * There is another path that exists: /testDir2/test4/test3
+ * On rename(/testDir2/test1/test2/test3, /testDir2/test4).
+ *
+ *
+ * Expectation for HNS / Blob endpoint:
+ * - Rename should fail
+ * - No file should be transferred to destination directory
+ *
+ */
+ @Test
+ public void testPosixRenameDirectoryWhereDirectoryAlreadyThereOnDestination()
+ throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ fs.mkdirs(new Path("testDir2/test1/test2/test3"));
+ fs.create(new Path("testDir2/test1/test2/test3/file"));
+ fs.mkdirs(new Path("testDir2/test4/test3"));
+ assertTrue(fs.exists(new Path("testDir2/test1/test2/test3/file")));
+ Assert.assertFalse(fs.rename(new Path("testDir2/test1/test2/test3"),
+ new Path("testDir2/test4")));
+ assertTrue(fs.exists(new Path("testDir2")));
+ assertTrue(fs.exists(new Path("testDir2/test1/test2")));
+ assertTrue(fs.exists(new Path("testDir2/test4")));
+ assertTrue(fs.exists(new Path("testDir2/test1/test2/test3")));
+ if (getIsNamespaceEnabled(fs)
+ || fs.getAbfsStore().getAbfsConfiguration().getPrefixMode()
+ == PrefixMode.BLOB) {
+ assertFalse(fs.exists(new Path("testDir2/test4/test3/file")));
+ assertTrue(fs.exists(new Path("testDir2/test1/test2/test3/file")));
+ } else {
+ assertTrue(fs.exists(new Path("testDir2/test4/test3/file")));
+ assertFalse(fs.exists(new Path("testDir2/test1/test2/test3/file")));
+ }
+ }
+
+ @Test
+ public void testPosixRenameDirectoryWherePartAlreadyThereOnDestination()
+ throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ fs.mkdirs(new Path("testDir2/test1/test2/test3"));
+ fs.create(new Path("testDir2/test1/test2/test3/file"));
+ fs.create(new Path("testDir2/test1/test2/test3/file1"));
+ fs.mkdirs(new Path("testDir2/test4/"));
+ fs.create(new Path("testDir2/test4/file1"));
+ byte[] etag = fs.getXAttr(new Path("testDir2/test4/file1"), "ETag");
+ assertTrue(fs.exists(new Path("testDir2/test1/test2/test3/file")));
+ assertTrue(fs.exists(new Path("testDir2/test1/test2/test3/file1")));
+ Assert.assertTrue(fs.rename(new Path("testDir2/test1/test2/test3"),
+ new Path("testDir2/test4")));
+ assertTrue(fs.exists(new Path("testDir2")));
+ assertTrue(fs.exists(new Path("testDir2/test1/test2")));
+ assertTrue(fs.exists(new Path("testDir2/test4")));
+ assertFalse(fs.exists(new Path("testDir2/test1/test2/test3")));
+
+
+ assertFalse(fs.exists(new Path("testDir2/test4/file")));
+ assertTrue(fs.exists(new Path("testDir2/test4/file1")));
+ assertTrue(fs.exists(new Path("testDir2/test4/test3/file")));
+ assertTrue(fs.exists(new Path("testDir2/test4/test3/file1")));
+ assertTrue(fs.exists(new Path("testDir2/test4/file1")));
+ assertFalse(fs.exists(new Path("testDir2/test1/test2/test3/file")));
+ assertFalse(fs.exists(new Path("testDir2/test1/test2/test3/file1")));
+ }
+
+ private void assumeNonHnsAccountBlobEndpoint(final AzureBlobFileSystem fs) {
+ Assume.assumeTrue("To work on only on non-HNS Blob endpoint",
+ fs.getAbfsStore().getAbfsConfiguration().getPrefixMode()
+ == PrefixMode.BLOB);
+ }
+
+ /**
+ * Test that after completing rename for a directory which is enabled for
+ * AtomicRename, the RenamePending JSON file is deleted.
+ */
+ @Test
+ public void testRenamePendingJsonIsRemovedPostSuccessfulRename()
+ throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ assumeNonHnsAccountBlobEndpoint(fs);
+ fs.setWorkingDirectory(new Path("/"));
+ fs.mkdirs(new Path("hbase/test1/test2/test3"));
+ fs.create(new Path("hbase/test1/test2/test3/file"));
+ fs.create(new Path("hbase/test1/test2/test3/file1"));
+ fs.mkdirs(new Path("hbase/test4/"));
+ fs.create(new Path("hbase/test4/file1"));
+ final AzureBlobFileSystem spiedFs = Mockito.spy(fs);
+ final Integer[] correctDeletePathCount = new Integer[1];
+ correctDeletePathCount[0] = 0;
+
+ Mockito.doAnswer(answer -> {
+ final String correctDeletePath = "/hbase/test1/test2/test3" + SUFFIX;
+ if (correctDeletePath.equals(
+ ((Path) answer.getArgument(0)).toUri().getPath())) {
+ correctDeletePathCount[0] = 1;
+ }
+ return null;
+ }).when(spiedFs).delete(Mockito.any(Path.class), Mockito.anyBoolean());
+ Assert.assertTrue(spiedFs.rename(new Path("hbase/test1/test2/test3"),
+ new Path("hbase/test4")));
+ Assert.assertTrue(correctDeletePathCount[0] == 1);
+ }
+
+ /**
+ * Test for a directory in /hbase directory. To simulate the crash of process,
+ * test will throw an exception with 503 on a copy of one of the blob.
+ * ListStatus API will be called on the directory. Expectation is that the ListStatus
+ * API of {@link AzureBlobFileSystem} should recover the paused rename.
+ */
+ @Test
+ public void testHBaseHandlingForFailedRename() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ assumeNonHnsAccountBlobEndpoint(fs);
+ final String failedCopyPath = "hbase/test1/test2/test3/file1";
+ fs.setWorkingDirectory(new Path("/"));
+ fs.mkdirs(new Path("hbase/test1/test2/test3"));
+ fs.create(new Path("hbase/test1/test2/test3/file"));
+ fs.create(new Path(failedCopyPath));
+ fs.mkdirs(new Path("hbase/test4/"));
+ fs.create(new Path("hbase/test4/file1"));
+ final AzureBlobFileSystem spiedFs = Mockito.spy(fs);
+ final AzureBlobFileSystemStore spiedAbfsStore = Mockito.spy(
+ spiedFs.getAbfsStore());
+ Mockito.doReturn(spiedAbfsStore).when(spiedFs).getAbfsStore();
+ AbfsClient spiedClient = Mockito.spy(spiedAbfsStore.getClient());
+ spiedAbfsStore.setClient(spiedClient);
+ Map pathLeaseIdMap = new HashMap<>();
+ Mockito.doAnswer(answer -> {
+ AbfsRestOperation op = (AbfsRestOperation) answer.callRealMethod();
+ String leaseId = op.getResult().getResponseHeader(X_MS_LEASE_ID);
+ pathLeaseIdMap.put(answer.getArgument(0), leaseId);
+ return op;
+ }).when(spiedClient).acquireBlobLease(Mockito.anyString(), Mockito.anyInt(), Mockito.any(TracingContext.class));
+ final Integer[] correctDeletePathCount = new Integer[1];
+ correctDeletePathCount[0] = 0;
+
+ //fail copy of /hbase/test1/test2/test3/file1.
+ Mockito.doAnswer(answer -> {
+ final Path srcPath = answer.getArgument(0);
+ final Path dstPath = answer.getArgument(1);
+ final String leaseId = answer.getArgument(2);
+ final TracingContext tracingContext = answer.getArgument(3);
+ if (("/" + failedCopyPath).equalsIgnoreCase(srcPath.toUri().getPath())) {
+ throw new AbfsRestOperationException(HttpURLConnection.HTTP_UNAVAILABLE,
+ AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT.getErrorCode(),
+ "Ingress is over the account limit.", new Exception());
+ }
+ fs.getAbfsStore().copyBlob(srcPath, dstPath, leaseId, tracingContext);
+ return null;
+ })
+ .when(spiedAbfsStore)
+ .copyBlob(Mockito.any(Path.class), Mockito.any(Path.class),
+ Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+ try {
+ spiedFs.rename(new Path("hbase/test1/test2/test3"),
+ new Path("hbase/test4"));
+ } catch (Exception ex) {
+
+ }
+ Assert.assertTrue(fs.exists(new Path(failedCopyPath)));
+ Assert.assertFalse(spiedFs.exists(new Path(
+ failedCopyPath.replace("test1/test2/test3/", "test4/test3/"))));
+
+ //call listPath API, it will recover the rename atomicity.
+ final AzureBlobFileSystem spiedFsForListPath = Mockito.spy(fs);
+ final AzureBlobFileSystemStore spiedStoreForListPath = Mockito.spy(
+ spiedFsForListPath.getAbfsStore());
+ Mockito.doReturn(spiedStoreForListPath)
+ .when(spiedFsForListPath)
+ .getAbfsStore();
+
+ /*
+ * Check if the fs.delete is on the renameJson file.
+ */
+ AtomicInteger deletedCount = new AtomicInteger(0);
+ AtomicBoolean srcSuffixDeletion = new AtomicBoolean(false);
+ Mockito.doAnswer(answer -> {
+ Path path = answer.getArgument(0);
+ Boolean recursive = answer.getArgument(1);
+ Assert.assertTrue(
+ ("/" + "hbase/test1/test2/test3" + SUFFIX).equalsIgnoreCase(
+ path.toUri().getPath()));
+ srcSuffixDeletion.set(true);
+ deletedCount.incrementAndGet();
+ return fs.delete(path, recursive);
+ })
+ .when(spiedFsForListPath)
+ .delete(Mockito.any(Path.class), Mockito.anyBoolean());
+
+ /*
+ * Check if the blob which will be retried is deleted from the renameBlob
+ * method.
+ */
+ AbfsClient client = spiedFsForListPath.getAbfsClient();
+ final AbfsClient spiedClientForListPath = Mockito.spy(client);
+ spiedFsForListPath.getAbfsStore().setClient(spiedClientForListPath);
+ Mockito.doAnswer(answer -> {
+ Path path = answer.getArgument(0);
+ String leaseId = answer.getArgument(1);
+ TracingContext tracingContext = answer.getArgument(2);
+ if (srcSuffixDeletion.get()) {
+ Assert.assertTrue(
+ ("/" + "hbase/test1/test2/test3" + SUFFIX).equalsIgnoreCase(
+ path.toUri().getPath()));
+ } else {
+ Assert.assertTrue(
+ ("/" + failedCopyPath).equalsIgnoreCase(path.toUri().getPath())
+ || "/hbase/test1/test2/test3".equalsIgnoreCase(
+ path.toUri().getPath()));
+ deletedCount.incrementAndGet();
+ }
+ client.deleteBlobPath(path, leaseId, tracingContext);
+ return null;
+ })
+ .when(spiedClientForListPath)
+ .deleteBlobPath(Mockito.any(Path.class),
+ Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+
+ for(Map.Entry entry : pathLeaseIdMap.entrySet()) {
+ try {
+ fs.getAbfsClient()
+ .releaseBlobLease(entry.getKey(), entry.getValue(),
+ getTestTracingContext(fs, true));
+ } catch (AbfsRestOperationException ex) {
+
+ }
+ }
+ spiedFsForListPath.listStatus(new Path("hbase/test1/test2"));
+ /*
+ * The invocation of getPathProperty will happen on the reinvocation of listStatus
+ * of /hbase/test2/test3 as after the resume, there will be no listing for the path
+ * and hence, getFileStatus would be required.
+ */
+ Mockito.verify(spiedStoreForListPath, Mockito.times(1))
+ .getPathProperty(Mockito.any(Path.class),
+ Mockito.any(TracingContext.class), Mockito.anyBoolean());
+ Assert.assertTrue(deletedCount.get() == 3);
+ Assert.assertFalse(spiedFsForListPath.exists(new Path(failedCopyPath)));
+ Assert.assertTrue(spiedFsForListPath.exists(new Path(
+ failedCopyPath.replace("test1/test2/test3/", "test4/test3/"))));
+ }
+
+ /**
+ * Test for a directory in /hbase directory. To simulate the crash of process,
+ * test will throw an exception with 503 on a copy of one of the blob. The
+ * source directory is a nested directory.
+ * ListStatus API will be called on the directory. Expectation is that the ListStatus
+ * API of {@link AzureBlobFileSystem} should recover the paused rename.
+ */
+ @Test
+ public void testHBaseHandlingForFailedRenameForNestedSourceThroughListFile()
+ throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ assumeNonHnsAccountBlobEndpoint(fs);
+ final String failedCopyPath = "hbase/test1/test2/test3/file1";
+ fs.setWorkingDirectory(new Path("/"));
+ fs.mkdirs(new Path("hbase/test1/test2/test3"));
+ fs.create(new Path("hbase/test1/test2/test3/file"));
+ fs.create(new Path(failedCopyPath));
+ fs.mkdirs(new Path("hbase/test4/"));
+ fs.create(new Path("hbase/test4/file1"));
+ final AzureBlobFileSystem spiedFs = Mockito.spy(fs);
+ final AzureBlobFileSystemStore spiedAbfsStore = Mockito.spy(
+ spiedFs.getAbfsStore());
+ Mockito.doReturn(spiedAbfsStore).when(spiedFs).getAbfsStore();
+ AbfsClient spiedClient = Mockito.spy(spiedAbfsStore.getClient());
+ spiedAbfsStore.setClient(spiedClient);
+ Map pathLeaseIdMap = new HashMap<>();
+ Mockito.doAnswer(answer -> {
+ AbfsRestOperation op = (AbfsRestOperation) answer.callRealMethod();
+ String leaseId = op.getResult().getResponseHeader(X_MS_LEASE_ID);
+ pathLeaseIdMap.put(answer.getArgument(0), leaseId);
+ return op;
+ }).when(spiedClient).acquireBlobLease(Mockito.anyString(), Mockito.anyInt(), Mockito.any(TracingContext.class));
+ final Integer[] correctDeletePathCount = new Integer[1];
+ correctDeletePathCount[0] = 0;
+
+ //fail copy of /hbase/test1/test2/test3/file1.
+ Mockito.doAnswer(answer -> {
+ final Path srcPath = answer.getArgument(0);
+ final Path dstPath = answer.getArgument(1);
+ final String leaseId = answer.getArgument(2);
+ final TracingContext tracingContext = answer.getArgument(3);
+ if (("/" + failedCopyPath).equalsIgnoreCase(srcPath.toUri().getPath())) {
+ throw new AbfsRestOperationException(HttpURLConnection.HTTP_UNAVAILABLE,
+ AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT.getErrorCode(),
+ "Ingress is over the account limit.", new Exception());
+ }
+ fs.getAbfsStore().copyBlob(srcPath, dstPath, leaseId, tracingContext);
+ return null;
+ })
+ .when(spiedAbfsStore)
+ .copyBlob(Mockito.any(Path.class), Mockito.any(Path.class),
+ Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+ try {
+ spiedFs.rename(new Path("hbase/test1/test2"),
+ new Path("hbase/test4"));
+ } catch (Exception ex) {
+
+ }
+ Assert.assertTrue(fs.exists(new Path(failedCopyPath)));
+ Assert.assertFalse(spiedFs.exists(new Path(
+ failedCopyPath.replace("test1/test2/test3/", "test4/test3/"))));
+
+ //call listPath API, it will recover the rename atomicity.
+ final AzureBlobFileSystem spiedFsForListPath = Mockito.spy(fs);
+ final AzureBlobFileSystemStore spiedStoreForListPath = Mockito.spy(
+ spiedFsForListPath.getAbfsStore());
+ Mockito.doReturn(spiedStoreForListPath)
+ .when(spiedFsForListPath)
+ .getAbfsStore();
+
+ /*
+ * Check if the fs.delete is on the renameJson file.
+ */
+ AtomicInteger deletedCount = new AtomicInteger(0);
+ AtomicBoolean srcDirSuffixDeletion = new AtomicBoolean(false);
+ Mockito.doAnswer(answer -> {
+ Path path = answer.getArgument(0);
+ Boolean recursive = answer.getArgument(1);
+ Assert.assertTrue(("/" + "hbase/test1/test2" + SUFFIX).equalsIgnoreCase(
+ path.toUri().getPath()));
+ srcDirSuffixDeletion.set(true);
+ deletedCount.incrementAndGet();
+ return fs.delete(path, recursive);
+ })
+ .when(spiedFsForListPath)
+ .delete(Mockito.any(Path.class), Mockito.anyBoolean());
+
+ /*
+ * Check if the blob which will be retried is deleted from the renameBlob
+ * method.
+ */
+ AbfsClient client = spiedFsForListPath.getAbfsClient();
+ final AbfsClient spiedClientForListPath = Mockito.spy(client);
+ spiedFsForListPath.getAbfsStore().setClient(spiedClientForListPath);
+ Mockito.doAnswer(answer -> {
+ Path path = answer.getArgument(0);
+ String leaseId = answer.getArgument(1);
+ TracingContext tracingContext = answer.getArgument(2);
+ if (srcDirSuffixDeletion.get()) {
+ Assert.assertTrue(("/" + "hbase/test1/test2" + SUFFIX).equalsIgnoreCase(
+ path.toUri().getPath()));
+ } else {
+ Assert.assertTrue(
+ ("/" + failedCopyPath).equalsIgnoreCase(path.toUri().getPath())
+ || "/hbase/test1/test2".equalsIgnoreCase(
+ path.toUri().getPath()));
+ deletedCount.incrementAndGet();
+ }
+ client.deleteBlobPath(path, leaseId, tracingContext);
+ return null;
+ })
+ .when(spiedClientForListPath)
+ .deleteBlobPath(Mockito.any(Path.class),
+ Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+
+ /*
+ * listFile on /hbase/test1 would give no result because
+ * /hbase/test1/test2 would be totally moved to /hbase/test4.
+ *
+ */
+ for(Map.Entry entry : pathLeaseIdMap.entrySet()) {
+ try {
+ fs.getAbfsClient()
+ .releaseBlobLease(entry.getKey(), entry.getValue(),
+ getTestTracingContext(fs, true));
+ } catch (AbfsRestOperationException ex) {
+
+ }
+ }
+ final FileStatus[] listFileResult = spiedFsForListPath.listStatus(
+ new Path("hbase/test1"));
+ /*
+ * The invocation of getPathProperty will happen on the reinvocation of listStatus
+ * of /hbase/test2/test3 as after the resume, there will be no listing for the path
+ * and hence, getFileStatus would be required.
+ */
+ Mockito.verify(spiedStoreForListPath, Mockito.times(1))
+ .getPathProperty(Mockito.any(Path.class),
+ Mockito.any(TracingContext.class), Mockito.anyBoolean());
+ Assert.assertTrue(deletedCount.get() == 3);
+ Assert.assertFalse(spiedFsForListPath.exists(new Path(failedCopyPath)));
+ Assert.assertTrue(spiedFsForListPath.exists(new Path(
+ failedCopyPath.replace("test1/test2/test3/", "test4/test2/test3/"))));
+ Assert.assertTrue(listFileResult.length == 0);
+ }
+
+ /**
+ * Test for a directory in /hbase directory. To simulate the crash of process,
+ * test will throw an exception with 503 on a copy of one of the blob. The
+ * source directory is a nested directory.
+ * GetFileStatus API will be called on the directory. Expectation is that the
+ * GetFileStatus API of {@link AzureBlobFileSystem} should recover the paused
+ * rename.
+ */
+ @Test
+ public void testHBaseHandlingForFailedRenameForNestedSourceThroughGetPathStatus()
+ throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ assumeNonHnsAccountBlobEndpoint(fs);
+ final String failedCopyPath = "hbase/test1/test2/test3/file1";
+ fs.setWorkingDirectory(new Path("/"));
+ fs.mkdirs(new Path("hbase/test1/test2/test3"));
+ fs.create(new Path("hbase/test1/test2/test3/file"));
+ fs.create(new Path(failedCopyPath));
+ fs.mkdirs(new Path("hbase/test4/"));
+ fs.create(new Path("hbase/test4/file1"));
+ final AzureBlobFileSystem spiedFs = Mockito.spy(fs);
+ final AzureBlobFileSystemStore spiedAbfsStore = Mockito.spy(
+ spiedFs.getAbfsStore());
+ Mockito.doReturn(spiedAbfsStore).when(spiedFs).getAbfsStore();
+ AbfsClient spiedClient = Mockito.spy(spiedAbfsStore.getClient());
+ spiedAbfsStore.setClient(spiedClient);
+ Map pathLeaseIdMap = new HashMap<>();
+ Mockito.doAnswer(answer -> {
+ AbfsRestOperation op = (AbfsRestOperation) answer.callRealMethod();
+ String leaseId = op.getResult().getResponseHeader(X_MS_LEASE_ID);
+ pathLeaseIdMap.put(answer.getArgument(0), leaseId);
+ return op;
+ }).when(spiedClient).acquireBlobLease(Mockito.anyString(), Mockito.anyInt(), Mockito.any(TracingContext.class));
+ final Integer[] correctDeletePathCount = new Integer[1];
+ correctDeletePathCount[0] = 0;
+
+ //fail copy of /hbase/test1/test2/test3/file1.
+ Mockito.doAnswer(answer -> {
+ final Path srcPath = answer.getArgument(0);
+ final Path dstPath = answer.getArgument(1);
+ final String leaseId = answer.getArgument(2);
+ final TracingContext tracingContext = answer.getArgument(3);
+ if (("/" + failedCopyPath).equalsIgnoreCase(srcPath.toUri().getPath())) {
+ throw new AbfsRestOperationException(HttpURLConnection.HTTP_UNAVAILABLE,
+ AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT.getErrorCode(),
+ "Ingress is over the account limit.", new Exception());
+ }
+ fs.getAbfsStore().copyBlob(srcPath, dstPath, leaseId, tracingContext);
+ return null;
+ })
+ .when(spiedAbfsStore)
+ .copyBlob(Mockito.any(Path.class), Mockito.any(Path.class),
+ Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+ try {
+ spiedFs.rename(new Path("hbase/test1/test2"),
+ new Path("hbase/test4"));
+ } catch (Exception ex) {
+
+ }
+ Assert.assertTrue(fs.exists(new Path(failedCopyPath)));
+ Assert.assertFalse(spiedFs.exists(new Path(
+ failedCopyPath.replace("test1/test2/test3/", "test4/test3/"))));
+
+ //call listPath API, it will recover the rename atomicity.
+ final AzureBlobFileSystem spiedFsForListPath = Mockito.spy(fs);
+ final AzureBlobFileSystemStore spiedStoreForListPath = Mockito.spy(
+ spiedFsForListPath.getAbfsStore());
+ Mockito.doReturn(spiedStoreForListPath)
+ .when(spiedFsForListPath)
+ .getAbfsStore();
+
+ /*
+ * Check if the fs.delete is on the renameJson file.
+ */
+ AtomicInteger deletedCount = new AtomicInteger(0);
+ AtomicBoolean srcSuffixDeletion = new AtomicBoolean(false);
+ Mockito.doAnswer(answer -> {
+ Path path = answer.getArgument(0);
+ Boolean recursive = answer.getArgument(1);
+ Assert.assertTrue(("/" + "hbase/test1/test2" + SUFFIX).equalsIgnoreCase(
+ path.toUri().getPath()));
+ srcSuffixDeletion.set(true);
+ deletedCount.incrementAndGet();
+ return fs.delete(path, recursive);
+ })
+ .when(spiedFsForListPath)
+ .delete(Mockito.any(Path.class), Mockito.anyBoolean());
+
+ /*
+ * Check if the blob which will be retried is deleted from the renameBlob
+ * method.
+ */
+ AbfsClient client = spiedFsForListPath.getAbfsClient();
+ final AbfsClient spiedClientForListPath = Mockito.spy(client);
+ spiedFsForListPath.getAbfsStore().setClient(spiedClientForListPath);
+ Mockito.doAnswer(answer -> {
+ Path path = answer.getArgument(0);
+ String leaseId = answer.getArgument(1);
+ TracingContext tracingContext = answer.getArgument(2);
+ if (srcSuffixDeletion.get()) {
+ Assert.assertTrue(("/" + "hbase/test1/test2" + SUFFIX).equalsIgnoreCase(
+ path.toUri().getPath()));
+ } else {
+ Assert.assertTrue(
+ ("/" + failedCopyPath).equalsIgnoreCase(path.toUri().getPath())
+ || "/hbase/test1/test2".equalsIgnoreCase(
+ path.toUri().getPath()));
+ deletedCount.incrementAndGet();
+ }
+ client.deleteBlobPath(path, leaseId, tracingContext);
+ return null;
+ })
+ .when(spiedClientForListPath)
+ .deleteBlobPath(Mockito.any(Path.class),
+ Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+
+ /*
+ * getFileStatus on /hbase/test2 should give NOT_FOUND exception, since,
+ * /hbase/test1/test2 was partially renamed. On the invocation of getFileStatus
+ * on the directory, the remaining rename will be made. And as the directory is renamed,
+ * the method should give NOT_FOUND exception.
+ */
+ for(Map.Entry entry : pathLeaseIdMap.entrySet()) {
+ try {
+ fs.getAbfsClient().releaseBlobLease(entry.getKey(), entry.getValue(), getTestTracingContext(fs, true));
+ } catch (AbfsRestOperationException ex) {
+
+ }
+ }
+ FileStatus fileStatus = null;
+ Boolean notFoundExceptionReceived = false;
+ try {
+ fileStatus = spiedFsForListPath.getFileStatus(
+ new Path("hbase/test1/test2"));
+ } catch (FileNotFoundException ex) {
+ notFoundExceptionReceived = true;
+
+ }
+ Assert.assertTrue(notFoundExceptionReceived);
+ Assert.assertNull(fileStatus);
+ /*
+ * GetPathProperty on store would be called to get FileStatus for srcDirectory
+ * and the corresponding renamePendingJson file.
+ */
+ Mockito.verify(spiedStoreForListPath, Mockito.times(2))
+ .getPathProperty(Mockito.any(Path.class),
+ Mockito.any(TracingContext.class), Mockito.anyBoolean());
+ Assert.assertTrue(deletedCount.get() == 3);
+ Assert.assertFalse(spiedFsForListPath.exists(new Path(failedCopyPath)));
+ Assert.assertTrue(spiedFsForListPath.exists(new Path(
+ failedCopyPath.replace("test1/test2/test3/", "test4/test2/test3/"))));
+ }
+
+ /**
+ * Simulates a scenario where HMaster in Hbase starts up and executes listStatus
+ * API on the directory that has to be renamed by some other executor-machine.
+ * The scenario is that RenamePending JSON is created but before it could be
+ * appended, it has been opened by the HMaster. The HMaster will delete it. The
+ * machine doing rename would have to recreate the JSON file.
+ * ref: issue
+ */
+ @Test
+ public void testHbaseListStatusBeforeRenamePendingFileAppendedWithIngressOnBlob()
+ throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ assumeNonHnsAccountBlobEndpoint(fs);
+ fs.setWorkingDirectory(new Path("/"));
+ testHbaseListStatusBeforeRenamePendingFileAppended(fs);
+ }
+
+ @Test
+ public void testHbaseListStatusBeforeRenamePendingFileAppendedWithIngressOnDFS()
+ throws Exception {
+ AzureBlobFileSystem fs = this.getFileSystem();
+ assumeNonHnsAccountBlobEndpoint(fs);
+
+
+ Configuration configuration = Mockito.spy(fs.getAbfsStore().getAbfsConfiguration().getRawConfiguration());
+ configuration.set(FS_AZURE_INGRESS_FALLBACK_TO_DFS, TRUE);
+ fs = (AzureBlobFileSystem) FileSystem.newInstance(configuration);
+ fs.setWorkingDirectory(new Path("/"));
+ testHbaseListStatusBeforeRenamePendingFileAppended(fs);
+ }
+
+ private void testHbaseListStatusBeforeRenamePendingFileAppended(final AzureBlobFileSystem fs) throws IOException {
+ final String failedCopyPath = "hbase/test1/test2/test3/file1";
+ fs.mkdirs(new Path("hbase/test1/test2/test3"));
+ fs.create(new Path("hbase/test1/test2/test3/file"));
+ fs.create(new Path(failedCopyPath));
+ fs.mkdirs(new Path("hbase/test4/"));
+ fs.create(new Path("hbase/test4/file1"));
+ final AzureBlobFileSystem spiedFs = Mockito.spy(fs);
+ final AzureBlobFileSystemStore spiedAbfsStore = Mockito.spy(
+ spiedFs.getAbfsStore());
+ Mockito.doReturn(spiedAbfsStore).when(spiedFs).getAbfsStore();
+ final Integer[] correctDeletePathCount = new Integer[1];
+ correctDeletePathCount[0] = 0;
+
+ Boolean[] renamePendingJsonCreated = new Boolean[1];
+ renamePendingJsonCreated[0] = false;
+ Boolean[] parallelListStatusCalledOnTheDirBeingRenamed = new Boolean[1];
+ parallelListStatusCalledOnTheDirBeingRenamed[0] = false;
+ Mockito.doAnswer(answer -> {
+ Path path = answer.getArgument(0);
+ Boolean recursive = answer.getArgument(1);
+ FSDataOutputStream outputStream = fs.create(path, recursive);
+ renamePendingJsonCreated[0] = true;
+ while (!parallelListStatusCalledOnTheDirBeingRenamed[0]) {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return outputStream;
+ }).when(spiedFs).create(Mockito.any(Path.class), Mockito.anyBoolean());
+
+ try {
+ new Thread(() -> {
+ //wait for the renamePending created;
+ while (!renamePendingJsonCreated[0]) {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ try {
+ spiedFs.listStatus(new Path("hbase/test1"));
+ parallelListStatusCalledOnTheDirBeingRenamed[0] = true;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }).start();
+ spiedFs.rename(new Path("hbase/test1/test2"),
+ new Path("hbase/test4"));
+ } catch (Exception ex) {
+
+ }
+ Assert.assertFalse(fs.exists(new Path(failedCopyPath)));
+ Assert.assertTrue(
+ spiedFs.exists(new Path(failedCopyPath.replace("test1/", "test4/"))));
+ }
+
+ @Test
+ public void testHbaseEmptyRenamePendingJsonDeletedBeforeListStatusCanDelete()
+ throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ assumeNonHnsAccountBlobEndpoint(fs);
+ final String failedCopyPath = "hbase/test1/test2/test3/file1";
+ fs.setWorkingDirectory(new Path("/"));
+ fs.mkdirs(new Path("hbase/test1/test2/test3"));
+ fs.create(new Path("hbase/test1/test2/test3/file"));
+ fs.create(new Path(failedCopyPath));
+ fs.mkdirs(new Path("hbase/test4/"));
+ fs.create(new Path("hbase/test4/file1"));
+ final AzureBlobFileSystem spiedFs = Mockito.spy(fs);
+ final AzureBlobFileSystemStore spiedStore = Mockito.spy(
+ spiedFs.getAbfsStore());
+ Mockito.doReturn(spiedStore).when(spiedFs).getAbfsStore();
+ final Integer[] correctDeletePathCount = new Integer[1];
+ correctDeletePathCount[0] = 0;
+
+ AzureBlobFileSystem listFileFs = Mockito.spy(fs);
+
+
+ Boolean[] renamePendingJsonCreated = new Boolean[1];
+ renamePendingJsonCreated[0] = false;
+ Boolean[] parallelListStatusCalledOnTheDirBeingRenamed = new Boolean[1];
+ parallelListStatusCalledOnTheDirBeingRenamed[0] = false;
+ Boolean[] parallelDeleteOfRenamePendingFileFromRenameFlow = new Boolean[1];
+ parallelDeleteOfRenamePendingFileFromRenameFlow[0] = false;
+ Mockito.doAnswer(answer -> {
+ Path path = answer.getArgument(0);
+ Boolean recursive = answer.getArgument(1);
+ FSDataOutputStream outputStream = fs.create(path, recursive);
+ renamePendingJsonCreated[0] = true;
+ while (!parallelListStatusCalledOnTheDirBeingRenamed[0]) {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return outputStream;
+ }).when(spiedFs).create(Mockito.any(Path.class), Mockito.anyBoolean());
+
+ Mockito.doAnswer(answer -> {
+ Path path = answer.getArgument(0);
+ Boolean recursive = answer.getArgument(1);
+ if (("/hbase/test1/test2" + SUFFIX).equalsIgnoreCase(
+ path.toUri().getPath())) {
+ while (!parallelListStatusCalledOnTheDirBeingRenamed[0]) {
+ Thread.sleep(100);
+ }
+ parallelDeleteOfRenamePendingFileFromRenameFlow[0] = true;
+ return fs.delete(path, recursive);
+ }
+ return fs.delete(path, recursive);
+ }).when(spiedFs).delete(Mockito.any(Path.class), Mockito.anyBoolean());
+
+ Mockito.doAnswer(answer -> {
+ Path path = answer.getArgument(0);
+ if (("/hbase/test1/test2" + SUFFIX).equalsIgnoreCase(
+ path.toUri().getPath())) {
+ FSDataInputStream inputStream = fs.open(path);
+ parallelListStatusCalledOnTheDirBeingRenamed[0] = true;
+ while (!parallelDeleteOfRenamePendingFileFromRenameFlow[0]) {
+ Thread.sleep(100);
+ }
+ return inputStream;
+ }
+ return fs.open(path);
+ }).when(listFileFs).open(Mockito.any(Path.class));
+
+ try {
+ new Thread(() -> {
+ //wait for the renamePending created;
+ while (!renamePendingJsonCreated[0]) {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ try {
+ listFileFs.listStatus(new Path("hbase/test1"));
+ parallelListStatusCalledOnTheDirBeingRenamed[0] = true;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }).start();
+ spiedFs.rename(new Path("hbase/test1/test2"),
+ new Path("hbase/test4"));
+ } catch (Exception ex) {
+
+ }
+ Assert.assertFalse(fs.exists(new Path(failedCopyPath)));
+ Assert.assertTrue(
+ spiedFs.exists(new Path(failedCopyPath.replace("test1/", "test4/"))));
+ }
+
+ @Test
+ public void testInvalidJsonForRenamePendingFile() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ assumeNonHnsAccountBlobEndpoint(fs);
+ fs.setWorkingDirectory(new Path("/"));
+ fs.mkdirs(new Path("hbase/test1/test2/test3"));
+ fs.create(new Path("hbase/test1/test2/test3/file"));
+ fs.create(new Path("hbase/test1/test2/test3/file1"));
+ FSDataOutputStream outputStream = fs.create(
+ new Path("hbase/test1/test2/test3" + SUFFIX));
+ outputStream.writeChars("{ some wrong json");
+ outputStream.flush();
+ outputStream.close();
+
+ fs.listStatus(new Path("hbase/test1/test2"));
+ Assert.assertFalse(fs.exists(new Path("hbase/test1/test2/test3" + SUFFIX)));
+ }
+
+ @Test
+ public void testEmptyDirRenameResolveFromGetFileStatus() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ assumeNonHnsAccountBlobEndpoint(fs);
+ String srcDir = "/hbase/test1/test2/test3";
+ fs.setWorkingDirectory(new Path("/"));
+ fs.mkdirs(new Path(srcDir));
+ fs.create(new Path(srcDir, "file1"));
+ fs.mkdirs(new Path("hbase/test4"));
+
+ AzureBlobFileSystem spiedFs = Mockito.spy(fs);
+
+ AzureBlobFileSystemStore spiedAbfsStore = Mockito.spy(
+ spiedFs.getAbfsStore());
+ Mockito.doReturn(spiedAbfsStore).when(spiedFs).getAbfsStore();
+ AbfsClient spiedClient = Mockito.spy(spiedAbfsStore.getClient());
+ spiedAbfsStore.setClient(spiedClient);
+ Map pathLeaseIdMap = new HashMap<>();
+ Mockito.doAnswer(answer -> {
+ AbfsRestOperation op = (AbfsRestOperation) answer.callRealMethod();
+ String leaseId = op.getResult().getResponseHeader(X_MS_LEASE_ID);
+ pathLeaseIdMap.put(answer.getArgument(0), leaseId);
+ return op;
+ }).when(spiedClient).acquireBlobLease(Mockito.anyString(), Mockito.anyInt(), Mockito.any(TracingContext.class));
+ Mockito.doAnswer(answer -> {
+ throw new AbfsRestOperationException(HttpURLConnection.HTTP_UNAVAILABLE,
+ AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT.getErrorCode(),
+ "Ingress is over the account limit.", new Exception());
+ })
+ .when(spiedAbfsStore)
+ .copyBlob(Mockito.any(Path.class), Mockito.any(Path.class),
+ Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+ try {
+ spiedFs.rename(new Path(srcDir),
+ new Path("hbase/test4"));
+ } catch (Exception ex) {
+ }
+
+ Assert.assertFalse(spiedFs.exists(
+ new Path(srcDir.replace("test1/test2/test3", "test4/test3/"))));
+
+ //call listPath API, it will recover the rename atomicity.
+ for(Map.Entry entry : pathLeaseIdMap.entrySet()) {
+ try {
+ fs.getAbfsClient()
+ .releaseBlobLease(entry.getKey(), entry.getValue(),
+ getTestTracingContext(fs, true));
+ } catch (Exception e) {}
+ }
+
+ final AzureBlobFileSystem spiedFsForListPath = Mockito.spy(fs);
+ final AzureBlobFileSystemStore spiedStoreForListPath = Mockito.spy(
+ spiedFsForListPath.getAbfsStore());
+ Mockito.doReturn(spiedStoreForListPath)
+ .when(spiedFsForListPath)
+ .getAbfsStore();
+
+ /*
+ * Check if the fs.delete is on the renameJson file.
+ */
+ AtomicInteger deletedCount = new AtomicInteger(0);
+ AtomicBoolean deletedSrcDirSuffix = new AtomicBoolean(false);
+ Mockito.doAnswer(answer -> {
+ Path path = answer.getArgument(0);
+ Boolean recursive = answer.getArgument(1);
+ Assert.assertTrue(
+ (srcDir + SUFFIX).equalsIgnoreCase(path.toUri().getPath()));
+ deletedSrcDirSuffix.set(true);
+ deletedCount.incrementAndGet();
+ return fs.delete(path, recursive);
+ })
+ .when(spiedFsForListPath)
+ .delete(Mockito.any(Path.class), Mockito.anyBoolean());
+
+ /*
+ * Check if the blob which will be retried is deleted from the renameBlob
+ * method.
+ */
+ AbfsClient client = spiedFsForListPath.getAbfsClient();
+ final AbfsClient spiedClientForListPath = Mockito.spy(client);
+ spiedFsForListPath.getAbfsStore().setClient(spiedClientForListPath);
+ Mockito.doAnswer(answer -> {
+ Path path = answer.getArgument(0);
+ String leaseId = answer.getArgument(1);
+ TracingContext tracingContext = answer.getArgument(2);
+ if (deletedSrcDirSuffix.get()) {
+ Assert.assertTrue(
+ (srcDir + SUFFIX).equalsIgnoreCase(path.toUri().getPath()));
+ } else {
+ Assert.assertTrue(
+ ((srcDir).equalsIgnoreCase(path.toUri().getPath()) || (srcDir
+ + "/file1").equalsIgnoreCase(path.toUri().getPath())));
+ deletedCount.incrementAndGet();
+ }
+ client.deleteBlobPath(path, leaseId, tracingContext);
+ return null;
+ })
+ .when(spiedClientForListPath)
+ .deleteBlobPath(Mockito.any(Path.class),
+ Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+
+ /*
+ * getFileStatus on /hbase/test2 should give NOT_FOUND exception, since,
+ * /hbase/test1/test2 was partially renamed. On the invocation of getFileStatus
+ * on the directory, the remaining rename will be made. And as the directory is renamed,
+ * the method should give NOT_FOUND exception.
+ */
+ FileStatus fileStatus = null;
+ Boolean notFoundExceptionReceived = false;
+ try {
+ fileStatus = spiedFsForListPath.getFileStatus(new Path(srcDir));
+ } catch (FileNotFoundException ex) {
+ notFoundExceptionReceived = true;
+
+ }
+ Assert.assertTrue(notFoundExceptionReceived);
+ Assert.assertNull(fileStatus);
+ /*
+ * GetFileStatus on store would be called to get FileStatus for srcDirectory
+ * and the corresponding renamePendingJson file.
+ */
+ Mockito.verify(spiedStoreForListPath, Mockito.times(2))
+ .getPathProperty(Mockito.any(Path.class),
+ Mockito.any(TracingContext.class), Mockito.anyBoolean());
+ Assert.assertTrue(deletedCount.get() == 3);
+ Assert.assertFalse(spiedFsForListPath.exists(new Path(srcDir)));
+ Assert.assertTrue(spiedFsForListPath.getFileStatus(
+ new Path(srcDir.replace("test1/test2/test3", "test4/test3/")))
+ .isDirectory());
+ }
+
+ @Test
+ public void testRenameBlobIdempotency() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ assumeNonHnsAccountBlobEndpoint(fs);
+ String srcDir = "/test1/test2/test3";
+ fs.mkdirs(new Path(srcDir));
+ fs.create(new Path(srcDir, "file1"));
+ fs.create(new Path(srcDir, "file2"));
+
+ fs.mkdirs(new Path("/test4"));
+
+ final AzureBlobFileSystem spiedFs = Mockito.spy(fs);
+ final AzureBlobFileSystemStore spiedStore = Mockito.spy(fs.getAbfsStore());
+ Mockito.doReturn(spiedStore).when(spiedFs).getAbfsStore();
+ final AbfsClient spiedClient = Mockito.spy(fs.getAbfsClient());
+ spiedStore.setClient(spiedClient);
+
+ /*
+ * First call to copyBlob for file1 will fail with connection-reset, but the
+ * backend has got the call. Retry of that API would give 409 error.
+ */
+ boolean[] hasBeenCalled = new boolean[1];
+ hasBeenCalled[0] = false;
+
+ boolean[] connectionResetThrown = new boolean[1];
+ connectionResetThrown[0] = false;
+
+ AbfsClientTestUtil.setMockAbfsRestOperationForCopyBlobOperation(spiedClient,
+ (spiedRestOp, actualCallMakerOp) -> {
+
+ Mockito.doAnswer(answer -> {
+ if (spiedRestOp.getUrl().toString().contains("file1")
+ && !hasBeenCalled[0]) {
+ hasBeenCalled[0] = true;
+ actualCallMakerOp.execute(answer.getArgument(0));
+ AbfsRestOperationTestUtil.addAbfsHttpOpProcessResponseMock(
+ spiedRestOp, (mockAbfsHttpOp, actualAbfsHttpOp) -> {
+ Mockito.doAnswer(sendRequestAnswer -> {
+ if (!connectionResetThrown[0]) {
+ connectionResetThrown[0] = true;
+ throw new SocketException("connection-reset");
+ }
+ spiedRestOp.signRequest(actualAbfsHttpOp,
+ sendRequestAnswer.getArgument(2));
+ actualAbfsHttpOp.sendRequest(
+ sendRequestAnswer.getArgument(0),
+ sendRequestAnswer.getArgument(1),
+ sendRequestAnswer.getArgument(2));
+ AbfsHttpOpTestUtil.setConnection(mockAbfsHttpOp,
+ actualAbfsHttpOp);
+ return mockAbfsHttpOp;
+ }).when(mockAbfsHttpOp)
+ .sendRequest(Mockito.nullable(byte[].class),
+ Mockito.anyInt(), Mockito.anyInt());
+
+ return mockAbfsHttpOp;
+ });
+ Mockito.doCallRealMethod()
+ .when(spiedRestOp)
+ .execute(Mockito.any(TracingContext.class));
+ spiedRestOp.execute(answer.getArgument(0));
+ return spiedRestOp;
+ } else {
+ actualCallMakerOp.execute(answer.getArgument(0));
+ AbfsRestOperationTestUtil.setResult(spiedRestOp,
+ actualCallMakerOp.getResult());
+ return actualCallMakerOp;
+ }
+ }).when(spiedRestOp).execute(Mockito.any(TracingContext.class));
+ return spiedRestOp;
+ });
+
+ spiedFs.setWorkingDirectory(new Path("/"));
+
+ Assert.assertTrue(spiedFs.rename(new Path(srcDir), new Path("/test4")));
+ Assert.assertTrue(spiedFs.exists(new Path("test4/test3/file1")));
+ Assert.assertTrue(spiedFs.exists(new Path("test4/test3/file2")));
+ Assert.assertTrue(hasBeenCalled[0]);
+ Assert.assertTrue(connectionResetThrown[0]);
+ }
+
+ @Test
+ public void testRenameBlobIdempotencyWhereDstIsCreatedFromSomeOtherProcess()
+ throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ assumeNonHnsAccountBlobEndpoint(fs);
+ String srcDir = "/test1/test2/test3";
+ fs.mkdirs(new Path(srcDir));
+ fs.create(new Path(srcDir, "file1"));
+ fs.create(new Path(srcDir, "file2"));
+
+ fs.mkdirs(new Path("/test4"));
+
+ final AzureBlobFileSystem spiedFs = Mockito.spy(fs);
+ final AzureBlobFileSystemStore spiedStore = Mockito.spy(fs.getAbfsStore());
+ Mockito.doReturn(spiedStore).when(spiedFs).getAbfsStore();
+ final AbfsClient spiedClient = Mockito.spy(fs.getAbfsClient());
+ spiedStore.setClient(spiedClient);
+
+ /*
+ * First call to copyBlob for file1 will fail with connection-reset, but the
+ * backend has got the call. Retry of that API would give 409 error.
+ */
+ boolean[] hasBeenCalled = new boolean[1];
+ hasBeenCalled[0] = false;
+
+ boolean[] connectionResetThrown = new boolean[1];
+ connectionResetThrown[0] = false;
+
+ AbfsClientTestUtil.setMockAbfsRestOperationForCopyBlobOperation(spiedClient,
+ (spiedRestOp, actualCallMakerOp) -> {
+
+ Mockito.doAnswer(answer -> {
+ if (spiedRestOp.getUrl().toString().contains("file1")
+ && !hasBeenCalled[0]) {
+ hasBeenCalled[0] = true;
+ fs.create(new Path("/test4/test3", "file1"));
+ AbfsRestOperationTestUtil.addAbfsHttpOpProcessResponseMock(
+ spiedRestOp, (mockAbfsHttpOp, actualAbfsHttpOp) -> {
+ Mockito.doAnswer(sendRequestAnswer -> {
+ if (!connectionResetThrown[0]) {
+ connectionResetThrown[0] = true;
+ throw new SocketException("connection-reset");
+ }
+ spiedRestOp.signRequest(actualAbfsHttpOp,
+ sendRequestAnswer.getArgument(2));
+ actualAbfsHttpOp.sendRequest(
+ sendRequestAnswer.getArgument(0),
+ sendRequestAnswer.getArgument(1),
+ sendRequestAnswer.getArgument(2));
+ AbfsHttpOpTestUtil.setConnection(mockAbfsHttpOp,
+ actualAbfsHttpOp);
+ return mockAbfsHttpOp;
+ }).when(mockAbfsHttpOp)
+ .sendRequest(Mockito.nullable(byte[].class),
+ Mockito.anyInt(), Mockito.anyInt());
+
+ return mockAbfsHttpOp;
+ });
+ Mockito.doCallRealMethod()
+ .when(spiedRestOp)
+ .execute(Mockito.any(TracingContext.class));
+ spiedRestOp.execute(answer.getArgument(0));
+ return spiedRestOp;
+ } else {
+ actualCallMakerOp.execute(answer.getArgument(0));
+ AbfsRestOperationTestUtil.setResult(spiedRestOp,
+ actualCallMakerOp.getResult());
+ return actualCallMakerOp;
+ }
+ }).when(spiedRestOp).execute(Mockito.any(TracingContext.class));
+ return spiedRestOp;
+ });
+
+ spiedFs.setWorkingDirectory(new Path("/"));
+
+ Boolean dstAlreadyThere = false;
+ try {
+ spiedFs.rename(new Path(srcDir), new Path("/test4"));
+ } catch (RuntimeException ex) {
+ if (ex.getMessage().contains(HttpURLConnection.HTTP_CONFLICT + "")) {
+ dstAlreadyThere = true;
+ }
+ }
+ Assert.assertTrue(dstAlreadyThere);
+ Assert.assertTrue(hasBeenCalled[0]);
+ Assert.assertTrue(connectionResetThrown[0]);
+ }
+
+ @Test
+ public void testRenameBlobIdempotencyWhereDstIsCopiedFromSomeOtherProcess()
+ throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ assumeNonHnsAccountBlobEndpoint(fs);
+ String srcDir = "/test1/test2/test3";
+ fs.mkdirs(new Path(srcDir));
+ fs.create(new Path(srcDir, "file1"));
+ fs.create(new Path(srcDir, "file2"));
+
+ fs.mkdirs(new Path("/test4"));
+
+ final AzureBlobFileSystem spiedFs = Mockito.spy(fs);
+ final AzureBlobFileSystemStore spiedStore = Mockito.spy(fs.getAbfsStore());
+ Mockito.doReturn(spiedStore).when(spiedFs).getAbfsStore();
+ final AbfsClient spiedClient = Mockito.spy(fs.getAbfsClient());
+ spiedStore.setClient(spiedClient);
+
+ /*
+ * First call to copyBlob for file1 will fail with connection-reset, but the
+ * backend has got the call. Retry of that API would give 409 error.
+ */
+ boolean[] hasBeenCalled = new boolean[1];
+ hasBeenCalled[0] = false;
+
+ boolean[] connectionResetThrown = new boolean[1];
+ connectionResetThrown[0] = false;
+
+ AbfsClientTestUtil.setMockAbfsRestOperationForCopyBlobOperation(spiedClient,
+ (spiedRestOp, actualCallMakerOp) -> {
+
+ Mockito.doAnswer(answer -> {
+ if (spiedRestOp.getUrl().toString().contains("file1")
+ && !hasBeenCalled[0]) {
+ hasBeenCalled[0] = true;
+ fs.create(new Path("/randomDir/test3/file1"));
+ fs.rename(new Path("/randomDir/test3/file1"),
+ new Path("/test4/test3/file1"));
+ AbfsRestOperationTestUtil.addAbfsHttpOpProcessResponseMock(
+ spiedRestOp, (mockAbfsHttpOp, actualAbfsHttpOp) -> {
+ Mockito.doAnswer(sendRequestAnswer -> {
+ if (!connectionResetThrown[0]) {
+ connectionResetThrown[0] = true;
+ throw new SocketException("connection-reset");
+ }
+ spiedRestOp.signRequest(actualAbfsHttpOp,
+ sendRequestAnswer.getArgument(2));
+ actualAbfsHttpOp.sendRequest(
+ sendRequestAnswer.getArgument(0),
+ sendRequestAnswer.getArgument(1),
+ sendRequestAnswer.getArgument(2));
+ AbfsHttpOpTestUtil.setConnection(mockAbfsHttpOp,
+ actualAbfsHttpOp);
+ return mockAbfsHttpOp;
+ }).when(mockAbfsHttpOp)
+ .sendRequest(Mockito.nullable(byte[].class),
+ Mockito.anyInt(), Mockito.anyInt());
+
+ return mockAbfsHttpOp;
+ });
+ Mockito.doCallRealMethod()
+ .when(spiedRestOp)
+ .execute(Mockito.any(TracingContext.class));
+ spiedRestOp.execute(answer.getArgument(0));
+ return spiedRestOp;
+ } else {
+ actualCallMakerOp.execute(answer.getArgument(0));
+ AbfsRestOperationTestUtil.setResult(spiedRestOp,
+ actualCallMakerOp.getResult());
+ return actualCallMakerOp;
+ }
+ }).when(spiedRestOp).execute(Mockito.any(TracingContext.class));
+ return spiedRestOp;
+ });
+
+ spiedFs.setWorkingDirectory(new Path("/"));
+
+ Boolean dstAlreadyThere = false;
+ try {
+ spiedFs.rename(new Path(srcDir), new Path("/test4"));
+ } catch (RuntimeException ex) {
+ if (ex.getMessage().contains(HttpURLConnection.HTTP_CONFLICT + "")) {
+ dstAlreadyThere = true;
+ }
+ }
+ Assert.assertTrue(dstAlreadyThere);
+ Assert.assertTrue(hasBeenCalled[0]);
+ Assert.assertTrue(connectionResetThrown[0]);
+ }
+
+ @Test
+ public void testRenameLargeNestedDir() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ assumeNonHnsAccountBlobEndpoint(fs);
+ String dir = "/";
+ for (int i = 0; i < 100; i++) {
+ dir += ("dir" + i + "/");
+ fs.mkdirs(new Path(dir));
+ }
+ fs.mkdirs(new Path("/dst"));
+ fs.rename(new Path("/dir0"), new Path("/dst"));
+ dir = "";
+ for (int i = 0; i < 100; i++) {
+ dir += ("dir" + i + "/");
+ Assert.assertTrue("" + i, fs.exists(new Path("/dst/" + dir)));
+ }
+ }
+
+ @Test
+ public void testRenameDirWhenMarkerBlobIsAbsent() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ assumeNonHnsAccountBlobEndpoint(fs);
+ fs.mkdirs(new Path("/test1"));
+ fs.mkdirs(new Path("/test1/test2"));
+ fs.mkdirs(new Path("/test1/test2/test3"));
+ fs.create(new Path("/test1/test2/test3/file"));
+
+ fs.getAbfsClient()
+ .deleteBlobPath(new Path("/test1/test2"),
+ null, getTestTracingContext(fs, true));
+ fs.mkdirs(new Path("/test4/test5"));
+ fs.rename(new Path("/test4"), new Path("/test1/test2"));
+
+ Assert.assertTrue(fs.exists(new Path("/test1/test2/test4/test5")));
+
+ fs.mkdirs(new Path("/test6"));
+ fs.rename(new Path("/test6"), new Path("/test1/test2/test4/test5"));
+ Assert.assertTrue(fs.exists(new Path("/test1/test2/test4/test5/test6")));
+
+ fs.getAbfsClient()
+ .deleteBlobPath(new Path("/test1/test2/test4/test5/test6"),
+ null, getTestTracingContext(fs, true));
+ fs.mkdirs(new Path("/test7"));
+ fs.create(new Path("/test7/file"));
+ fs.rename(new Path("/test7"), new Path("/test1/test2/test4/test5/test6"));
+ Assert.assertTrue(
+ fs.exists(new Path("/test1/test2/test4/test5/test6/file")));
+ }
+
+ @Test
+ public void testBlobRenameSrcDirHasNoMarker() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ assumeNonHnsAccountBlobEndpoint(fs);
+ fs.create(new Path("/test1/test2/file1"));
+ fs.getAbfsStore()
+ .getClient()
+ .deleteBlobPath(new Path("/test1"), null, getTestTracingContext(fs, true));
+ intercept(AbfsRestOperationException.class, () -> {
+ fs.getAbfsStore().getBlobProperty(new Path("/test1"),
+ getTestTracingContext(fs, true));
+ });
+ fs.mkdirs(new Path("/test2"));
+ fs.rename(new Path("/test1"), new Path("/test2"));
+ Assert.assertTrue(fs.getAbfsStore()
+ .getBlobProperty(new Path("/test2/test1"),
+ getTestTracingContext(fs, true)).getIsDirectory());
+ }
+
+ @Test
+ public void testCopyBlobTakeTime() throws Exception {
+ AzureBlobFileSystem fileSystem = Mockito.spy(getFileSystem());
+ assumeNonHnsAccountBlobEndpoint(fileSystem);
+ AzureBlobFileSystemStore store = Mockito.spy(fileSystem.getAbfsStore());
+ Mockito.doReturn(store).when(fileSystem).getAbfsStore();
+ AbfsClient client = store.getClient();
+ AbfsClient spiedClient = Mockito.spy(client);
+ store.setClient(spiedClient);
+
+ Mockito.doAnswer(answer -> {
+ AbfsRestOperation op = Mockito.spy((AbfsRestOperation) answer.callRealMethod());
+ AbfsHttpOperation httpOp = Mockito.spy(op.getResult());
+ Mockito.doReturn(COPY_STATUS_PENDING).when(httpOp).getResponseHeader(
+ HttpHeaderConfigurations.X_MS_COPY_STATUS);
+ Mockito.doReturn(httpOp).when(op).getResult();
+ return op;
+ }).when(spiedClient).copyBlob(Mockito.any(Path.class), Mockito.any(Path.class),
+ Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+ fileSystem.create(new Path("/test1/file"));
+ fileSystem.rename(new Path("/test1/file"), new Path("/test1/file2"));
+ Assert.assertTrue(fileSystem.exists(new Path("/test1/file2")));
+ Mockito.verify(store, Mockito.times(1))
+ .handleCopyInProgress(Mockito.any(Path.class),
+ Mockito.any(TracingContext.class), Mockito.any(String.class));
+ }
+
+ @Test
+ public void testCopyBlobTakeTimeAndEventuallyFail() throws Exception {
+ AzureBlobFileSystem fileSystem = Mockito.spy(getFileSystem());
+ assumeNonHnsAccountBlobEndpoint(fileSystem);
+ AzureBlobFileSystemStore store = Mockito.spy(fileSystem.getAbfsStore());
+ Mockito.doReturn(store).when(fileSystem).getAbfsStore();
+ AbfsClient client = store.getClient();
+ AbfsClient spiedClient = Mockito.spy(client);
+ store.setClient(spiedClient);
+
+ Mockito.doAnswer(answer -> {
+ AbfsRestOperation op = Mockito.spy((AbfsRestOperation) answer.callRealMethod());
+ AbfsHttpOperation httpOp = Mockito.spy(op.getResult());
+ Mockito.doReturn(COPY_STATUS_PENDING).when(httpOp).getResponseHeader(
+ HttpHeaderConfigurations.X_MS_COPY_STATUS);
+ Mockito.doReturn(httpOp).when(op).getResult();
+ return op;
+ }).when(spiedClient).copyBlob(Mockito.any(Path.class), Mockito.any(Path.class),
+ Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+ Mockito.doAnswer(answer -> {
+ AbfsRestOperation op = Mockito.spy((AbfsRestOperation) answer.callRealMethod());
+ AbfsHttpOperation httpOp = Mockito.spy(op.getResult());
+ Mockito.doReturn(COPY_STATUS_FAILED).when(httpOp).getResponseHeader(
+ HttpHeaderConfigurations.X_MS_COPY_STATUS);
+ Mockito.doReturn(httpOp).when(op).getResult();
+ return op;
+ }).when(spiedClient).getBlobProperty(Mockito.any(Path.class), Mockito.any(TracingContext.class));
+
+ fileSystem.create(new Path("/test1/file"));
+ Boolean copyBlobFailureCaught = false;
+ try {
+ fileSystem.rename(new Path("/test1/file"), new Path("/test1/file2"));
+ } catch (AbfsRestOperationException e) {
+ if (COPY_BLOB_FAILED.equals(e.getErrorCode())) {
+ copyBlobFailureCaught = true;
+ }
+ }
+ Assert.assertTrue(copyBlobFailureCaught);
+ Assert.assertTrue(fileSystem.exists(new Path("/test1/file")));
+ Mockito.verify(store, Mockito.times(1))
+ .handleCopyInProgress(Mockito.any(Path.class),
+ Mockito.any(TracingContext.class), Mockito.any(String.class));
+ }
+
+ @Test
+ public void testCopyBlobTakeTimeAndEventuallyAborted() throws Exception {
+ AzureBlobFileSystem fileSystem = Mockito.spy(getFileSystem());
+ assumeNonHnsAccountBlobEndpoint(fileSystem);
+ AzureBlobFileSystemStore store = Mockito.spy(fileSystem.getAbfsStore());
+ Mockito.doReturn(store).when(fileSystem).getAbfsStore();
+ AbfsClient client = store.getClient();
+ AbfsClient spiedClient = Mockito.spy(client);
+ store.setClient(spiedClient);
+
+ Mockito.doAnswer(answer -> {
+ AbfsRestOperation op = Mockito.spy((AbfsRestOperation) answer.callRealMethod());
+ AbfsHttpOperation httpOp = Mockito.spy(op.getResult());
+ Mockito.doReturn(COPY_STATUS_PENDING).when(httpOp).getResponseHeader(
+ HttpHeaderConfigurations.X_MS_COPY_STATUS);
+ Mockito.doReturn(httpOp).when(op).getResult();
+ return op;
+ }).when(spiedClient).copyBlob(Mockito.any(Path.class), Mockito.any(Path.class),
+ Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+ Mockito.doAnswer(answer -> {
+ AbfsRestOperation op = Mockito.spy((AbfsRestOperation) answer.callRealMethod());
+ AbfsHttpOperation httpOp = Mockito.spy(op.getResult());
+ Mockito.doReturn(COPY_STATUS_ABORTED).when(httpOp).getResponseHeader(
+ HttpHeaderConfigurations.X_MS_COPY_STATUS);
+ Mockito.doReturn(httpOp).when(op).getResult();
+ return op;
+ }).when(spiedClient).getBlobProperty(Mockito.any(Path.class), Mockito.any(TracingContext.class));
+
+ fileSystem.create(new Path("/test1/file"));
+ Boolean copyBlobFailureCaught = false;
+ try {
+ fileSystem.rename(new Path("/test1/file"), new Path("/test1/file2"));
+ } catch (AbfsRestOperationException e) {
+ if (COPY_BLOB_ABORTED.equals(e.getErrorCode())) {
+ copyBlobFailureCaught = true;
+ }
+ }
+ Assert.assertTrue(copyBlobFailureCaught);
+ Assert.assertTrue(fileSystem.exists(new Path("/test1/file")));
+ Mockito.verify(store, Mockito.times(1))
+ .handleCopyInProgress(Mockito.any(Path.class),
+ Mockito.any(TracingContext.class), Mockito.any(String.class));
+ }
+
+ @Test
+ public void testCopyBlobTakeTimeAndBlobIsDeleted() throws Exception {
+ AzureBlobFileSystem fileSystem = Mockito.spy(getFileSystem());
+ assumeNonHnsAccountBlobEndpoint(fileSystem);
+ AzureBlobFileSystemStore store = Mockito.spy(fileSystem.getAbfsStore());
+ String srcFile = "/test1/file";
+ String dstFile = "/test1/file2";
+ Mockito.doReturn(store).when(fileSystem).getAbfsStore();
+ AbfsClient client = store.getClient();
+ AbfsClient spiedClient = Mockito.spy(client);
+ store.setClient(spiedClient);
+
+ Mockito.doAnswer(answer -> {
+ AbfsRestOperation op = Mockito.spy((AbfsRestOperation) answer.callRealMethod());
+ fileSystem.delete(new Path(dstFile), false);
+ AbfsHttpOperation httpOp = Mockito.spy(op.getResult());
+ Mockito.doReturn(COPY_STATUS_PENDING).when(httpOp).getResponseHeader(
+ HttpHeaderConfigurations.X_MS_COPY_STATUS);
+ Mockito.doReturn(httpOp).when(op).getResult();
+ return op;
+ }).when(spiedClient).copyBlob(Mockito.any(Path.class), Mockito.any(Path.class),
+ Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+
+ fileSystem.create(new Path(srcFile));
+
+
+ Assert.assertFalse(fileSystem.rename(new Path(srcFile), new Path(dstFile)));
+ Assert.assertFalse(fileSystem.exists(new Path(dstFile)));
+ }
+
+ @Test
+ public void testParallelCopy() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ assumeNonHnsAccountBlobEndpoint(fs);
+ fs.create(new Path("/src"));
+ boolean[] dstBlobAlreadyThereExceptionReceived = new boolean[1];
+ dstBlobAlreadyThereExceptionReceived[0] = false;
+ AtomicInteger threadsCompleted = new AtomicInteger(0);
+ new Thread(() -> {
+ parallelCopyRunnable(fs, dstBlobAlreadyThereExceptionReceived,
+ threadsCompleted);
+ }).start();
+ new Thread(() -> {
+ parallelCopyRunnable(fs, dstBlobAlreadyThereExceptionReceived,
+ threadsCompleted);
+ }).start();
+ while (threadsCompleted.get() < 2) ;
+ Assert.assertTrue(dstBlobAlreadyThereExceptionReceived[0]);
+ }
+
+ private void parallelCopyRunnable(final AzureBlobFileSystem fs,
+ final boolean[] dstBlobAlreadyThereExceptionReceived,
+ final AtomicInteger threadsCompleted) {
+ try {
+ fs.getAbfsClient().copyBlob(new Path("/src"),
+ new Path("/dst"), null, getTestTracingContext(fs, true));
+ } catch (AbfsRestOperationException ex) {
+ if (ex.getStatusCode() == HttpURLConnection.HTTP_CONFLICT) {
+ dstBlobAlreadyThereExceptionReceived[0] = true;
+ }
+ } catch (
+ AzureBlobFileSystemException e) {
+ }
+ threadsCompleted.incrementAndGet();
+ }
+
+ @Test
+ public void testCopyAfterSourceHasBeenDeleted() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ assumeNonHnsAccountBlobEndpoint(fs);
+ fs.create(new Path("/src"));
+ TracingContext tracingContext = new TracingContext("clientCorrelationId",
+ "fileSystemId", FSOperationType.TEST_OP,
+ getConfiguration().getTracingHeaderFormat(),
+ null);
+ fs.getAbfsStore()
+ .getClient()
+ .deleteBlobPath(new Path("/src"), null, getTestTracingContext(fs, true));
+ Boolean srcBlobNotFoundExReceived = false;
+
+ tracingContext = new TracingContext(tracingContext);
+ try {
+ fs.getAbfsStore()
+ .copyBlob(new Path("/src"), new Path("/dst"),
+ null, getTestTracingContext(fs, true));
+ } catch (AbfsRestOperationException ex) {
+ if (ex.getStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) {
+ srcBlobNotFoundExReceived = true;
+ }
+ }
+ Assert.assertTrue(srcBlobNotFoundExReceived);
+ }
+
+ @Test
+ public void testParallelRenameForAtomicDirShouldFail() throws Exception {
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ assumeNonHnsAccountBlobEndpoint(fs);
+ fs.setWorkingDirectory(new Path("/"));
+ fs.mkdirs(new Path("/hbase/dir1"));
+ fs.create(new Path("/hbase/dir1/file1"));
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ store.setClient(client);
+ AtomicBoolean leaseAcquired = new AtomicBoolean(false);
+ AtomicBoolean exceptionOnParallelRename = new AtomicBoolean(false);
+ AtomicBoolean parallelThreadDone = new AtomicBoolean(false);
+ Mockito.doAnswer(answer -> {
+ AbfsRestOperation op = (AbfsRestOperation) answer.callRealMethod();
+ leaseAcquired.set(true);
+ while (!parallelThreadDone.get()) ;
+ return op;
+ })
+ .when(client)
+ .acquireBlobLease(Mockito.anyString(), Mockito.anyInt(),
+ Mockito.any(TracingContext.class));
+
+ new Thread(() -> {
+ while (!leaseAcquired.get()) ;
+ try {
+ fs.rename(new Path("/hbase/dir1/file1"), new Path("/hbase/dir2/"));
+ } catch (Exception e) {
+ if (e.getCause() instanceof AbfsRestOperationException
+ && ((AbfsRestOperationException) e.getCause()).getStatusCode()
+ == HttpURLConnection.HTTP_CONFLICT) {
+ exceptionOnParallelRename.set(true);
+ }
+ } finally {
+ parallelThreadDone.set(true);
+ }
+ }).start();
+ fs.rename(new Path("/hbase/dir1/file1"), new Path("/hbase/dir2/"));
+ while (!parallelThreadDone.get()) ;
+ Assert.assertTrue(exceptionOnParallelRename.get());
+ }
+
+ @Test
+ public void testParallelAppendToFileBeingCopiedInAtomicDirectory()
+ throws Exception {
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ assumeNonHnsAccountBlobEndpoint(fs);
+ fs.setWorkingDirectory(new Path("/"));
+ fs.mkdirs(new Path("/hbase/dir1"));
+ fs.create(new Path("/hbase/dir1/file1"));
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ store.setClient(client);
+ AtomicBoolean copyOfSrcFile = new AtomicBoolean(false);
+ AtomicBoolean parallelAppendDone = new AtomicBoolean(false);
+ AtomicBoolean exceptionCaught = new AtomicBoolean(false);
+
+ Mockito.doAnswer(answer -> {
+ answer.callRealMethod();
+ if ("/hbase/dir1/file1".equalsIgnoreCase(
+ ((Path) answer.getArgument(0)).toUri().getPath())) {
+ copyOfSrcFile.set(true);
+ while (!parallelAppendDone.get()) ;
+ }
+ return null;
+ })
+ .when(store)
+ .copyBlob(Mockito.any(Path.class), Mockito.any(Path.class),
+ Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+
+ FSDataOutputStream outputStream = fs.append(new Path("/hbase/dir1/file1"));
+
+ new Thread(() -> {
+ while (!copyOfSrcFile.get()) ;
+ try {
+ byte[] bytes = new byte[4 * ONE_MB];
+ new Random().nextBytes(bytes);
+ outputStream.write(bytes);
+ outputStream.hsync();
+ } catch (Exception e) {
+ if (e.getCause() instanceof AbfsRestOperationException
+ && ((AbfsRestOperationException) e.getCause()).getStatusCode()
+ == HttpURLConnection.HTTP_PRECON_FAILED) {
+ exceptionCaught.set(true);
+ }
+ } finally {
+ parallelAppendDone.set(true);
+ }
+ }).start();
+
+ fs.rename(new Path("/hbase/dir1"), new Path("/hbase/dir2"));
+
+ while (!parallelAppendDone.get()) ;
+ Assert.assertTrue(exceptionCaught.get());
+ }
+
+ @Test
+ public void testParallelBlobLeaseOnChildBlobInRenameSrcDir()
+ throws Exception {
+ assumeNonHnsAccountBlobEndpoint(getFileSystem());
+ AzureBlobFileSystem fs = Mockito.spy(
+ (AzureBlobFileSystem) FileSystem.newInstance(getRawConfiguration()));
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ Path srcDirPath = new Path("/hbase/testDir");
+ fs.mkdirs(srcDirPath);
+ fs.create(new Path(srcDirPath, "file1"));
+ fs.create(new Path(srcDirPath, "file2"));
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ AbfsClient client = store.getClient();
+ AbfsClient spiedClient = Mockito.spy(client);
+ store.setClient(spiedClient);
+
+ fs.getAbfsClient()
+ .acquireBlobLease("/hbase/testDir/file2", -1,
+ getTestTracingContext(fs, true));
+
+ AbfsLease[] leases = new AbfsLease[1];
+ Mockito.doAnswer(answer -> {
+ String path = answer.getArgument(0);
+ AbfsLease lease = (AbfsLease) answer.callRealMethod();
+ if (srcDirPath.toUri().getPath().equalsIgnoreCase(path)) {
+ lease = Mockito.spy(lease);
+ leases[0] = lease;
+ }
+ return lease;
+ })
+ .when(store)
+ .getBlobLease(Mockito.anyString(), Mockito.nullable(Integer.class),
+ Mockito.any(TracingContext.class));
+
+ Boolean renameFailed = false;
+ try {
+ fs.rename(srcDirPath, new Path("/hbase/newDir"));
+ } catch (Exception e) {
+ renameFailed = true;
+ }
+
+ Assertions.assertThat(renameFailed).isTrue();
+ Mockito.verify(leases[0], Mockito.times(1)).free();
+ }
+
+ @Test
+ public void testParallelCreateNonRecursiveToFilePartOfAtomicDirectoryInRename()
+ throws Exception {
+ Configuration configuration = Mockito.spy(getRawConfiguration());
+ configuration.set(FS_AZURE_LEASE_CREATE_NON_RECURSIVE, "true");
+ FileSystem fsCreate = FileSystem.newInstance(configuration);
+ AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.newInstance(configuration);
+ assumeNonHnsAccountBlobEndpoint(fs);
+ fs.setWorkingDirectory(new Path("/"));
+ fs.mkdirs(new Path("/hbase/dir1"));
+ fs.create(new Path("/hbase/dir1/file1"));
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ fs.getAbfsStore().setClient(client);
+ AtomicBoolean leaseAcquired = new AtomicBoolean(false);
+ AtomicBoolean parallelCreateDone = new AtomicBoolean(false);
+ AtomicBoolean exceptionCaught = new AtomicBoolean(false);
+ AtomicBoolean parallelRenameDone = new AtomicBoolean(false);
+
+ Mockito.doAnswer(answer -> {
+ AbfsRestOperation op = (AbfsRestOperation) answer.callRealMethod();
+ leaseAcquired.set(true);
+ while(!parallelCreateDone.get());
+ return op;
+ })
+ .when(client)
+ .acquireBlobLease(Mockito.anyString(), Mockito.anyInt(),
+ Mockito.any(TracingContext.class));
+
+ new Thread(() -> {
+ try {
+ fs.rename(new Path("/hbase/dir1"), new Path("/hbase/dir2"));
+ } catch (Exception e) {} finally {
+ parallelRenameDone.set(true);
+ }
+ }).start();
+
+ Path createNewFilePath = new Path("/hbase/dir1/file2");
+ while (!leaseAcquired.get()) ;
+ try {
+ fsCreate.createFile(createNewFilePath)
+ .overwrite(false)
+ .replication((short) 1)
+ .bufferSize(1024)
+ .blockSize(1024)
+ .build();
+ } catch (AbfsRestOperationException e) {
+ if (e.getStatusCode()
+ == HttpURLConnection.HTTP_CONFLICT) {
+ exceptionCaught.set(true);
+ }
+ } finally {
+ parallelCreateDone.set(true);
+ }
+
+
+ while (!parallelRenameDone.get()) ;
+ Assert.assertTrue(exceptionCaught.get());
+ }
+
+ @Test
+ public void testBlobRenameOfDirectoryHavingNeighborWithSamePrefix()
+ throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ assumeNonHnsAccountBlobEndpoint(fs);
+ fs.mkdirs(new Path("/testDir/dir"));
+ fs.mkdirs(new Path("/testDir/dirSamePrefix"));
+ fs.create(new Path("/testDir/dir/file1"));
+ fs.create(new Path("/testDir/dir/file2"));
+
+ fs.create(new Path("/testDir/dirSamePrefix/file1"));
+ fs.create(new Path("/testDir/dirSamePrefix/file2"));
+
+ fs.rename(new Path("/testDir/dir"), new Path("/testDir/dir2"));
+
+ Assertions.assertThat(fs.exists(new Path("/testDir/dirSamePrefix/file1")))
+ .isTrue();
+ Assertions.assertThat(fs.exists(new Path("/testDir/dir/file1")))
+ .isFalse();
+ Assertions.assertThat(fs.exists(new Path("/testDir/dir/file2")))
+ .isFalse();
+ Assertions.assertThat(fs.exists(new Path("/testDir/dir/")))
+ .isFalse();
+ }
+
+ @Test
+ public void testBlobRenameCancelRenewTimerForLeaseTakenInAtomicRename()
+ throws Exception {
+ AzureBlobFileSystem fs = Mockito.spy(
+ (AzureBlobFileSystem) FileSystem.newInstance(getRawConfiguration()));
+ assumeNonHnsAccountBlobEndpoint(fs);
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+
+ fs.mkdirs(new Path("/hbase/dir"));
+ fs.create(new Path("/hbase/dir/file1"));
+ fs.create(new Path("/hbase/dir/file2"));
+
+ final List leases = new ArrayList<>();
+ Mockito.doAnswer(answer -> {
+ AbfsBlobLease lease = Mockito.spy(
+ (AbfsBlobLease) answer.callRealMethod());
+ leases.add(lease);
+ return lease;
+ })
+ .when(store)
+ .getBlobLease(Mockito.anyString(), Mockito.nullable(Integer.class),
+ Mockito.any(TracingContext.class));
+
+ fs.rename(new Path("/hbase/dir"), new Path("/hbase/dir2"));
+
+ Assertions.assertThat(leases).hasSize(3);
+ for (AbfsBlobLease lease : leases) {
+ Mockito.verify(lease, Mockito.times(1)).cancelTimer();
+ }
+ }
+
+ @Test
+ public void testBlobRenameServerReturnsOneBlobPerList() throws Exception {
+ assumeNonHnsAccountBlobEndpoint(getFileSystem());
+ AzureBlobFileSystem fs = (AzureBlobFileSystem) Mockito.spy(FileSystem.newInstance(getRawConfiguration()));
+ fs.mkdirs(new Path("/testDir/"));
+ fs.create(new Path("/testDir/file1"));
+ fs.create(new Path("/testDir/file2"));
+
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ AbfsClient client = fs.getAbfsClient();
+ AbfsClient spiedClient = Mockito.spy(client);
+ store.setClient(spiedClient);
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ Mockito.doAnswer(answer -> {
+ String marker = answer.getArgument(0);
+ String prefix = answer.getArgument(1);
+ String delimeter = answer.getArgument(2);
+ Integer count = answer.getArgument(3);
+ TracingContext tracingContext = answer.getArgument(4);
+ AbfsRestOperation op = client.getListBlobs(marker, prefix, delimeter, 1, tracingContext);
+ return op;
+ }).when(spiedClient).getListBlobs(Mockito.nullable(String.class),
+ Mockito.nullable(String.class), Mockito.nullable(String.class),
+ Mockito.nullable(Integer.class), Mockito.any(TracingContext.class));
+
+ fs.rename(new Path("/testDir"), new Path("/testDir1"));
+ Assertions.assertThat(fs.exists(new Path("/testDir"))).isFalse();
+ Assertions.assertThat(fs.exists(new Path("/testDir1"))).isTrue();
+ Assertions.assertThat(fs.exists(new Path("/testDir1/file1"))).isTrue();
+ Assertions.assertThat(fs.exists(new Path("/testDir1/file2"))).isTrue();
+ }
+
+ @Test
+ public void testBlobAtomicRenameSrcAndDstAreNotLeftLeased() throws Exception {
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ assumeNonHnsAccountBlobEndpoint(fs);
+ Assume.assumeFalse(getConfiguration().shouldIngressFallbackToDfs());
+ fs.setWorkingDirectory(new Path("/"));
+ fs.create(new Path("/hbase/dir1/blob1"));
+ fs.create(new Path("/hbase/dir1/blob2"));
+ fs.rename(new Path("/hbase/dir1/"), new Path("/hbase/dir2"));
+ fs.create(new Path("/hbase/dir1/blob1"));
+ byte[] bytes = new byte[4 * ONE_MB];
+ new Random().nextBytes(bytes);
+ try (FSDataOutputStream os = fs.append(new Path("hbase/dir2/blob1"))) {
+ os.write(bytes);
+ }
+ }
+
+ /**
+ * Test to assert that the CID in src marker blob copy and delete contains the
+ * total number of blobs operated in the rename directory.
+ * Also, to assert that all operations in the rename-directory flow have same
+ * primaryId and opType.
+ */
+ @Test
+ public void testRenameSrcDirDeleteEmitDeletionCountInClientRequestId()
+ throws Exception {
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ Assume.assumeTrue(getPrefixMode(fs) == PrefixMode.BLOB);
+ String dirPathStr = "/testDir/dir1";
+ fs.mkdirs(new Path(dirPathStr));
+ ExecutorService executorService = Executors.newFixedThreadPool(5);
+ List futures = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ final int iter = i;
+ Future future = executorService.submit(() -> {
+ return fs.create(new Path("/testDir/dir1/file" + iter));
+ });
+ futures.add(future);
+ }
+
+ for (Future future : futures) {
+ future.get();
+ }
+ executorService.shutdown();
+
+
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ AbfsClient client = Mockito.spy(store.getClient());
+ store.setClient(client);
+
+ final TracingHeaderValidator tracingHeaderValidator
+ = new TracingHeaderValidator(
+ fs.getAbfsStore().getAbfsConfiguration().getClientCorrelationId(),
+ fs.getFileSystemId(), FSOperationType.RENAME, true, 0);
+ fs.registerListener(tracingHeaderValidator);
+
+ Mockito.doAnswer(answer -> {
+ Mockito.doAnswer(copyAnswer -> {
+ if (dirPathStr.equalsIgnoreCase(
+ ((Path) copyAnswer.getArgument(0)).toUri().getPath())) {
+ tracingHeaderValidator.setOperatedBlobCount(11);
+ return copyAnswer.callRealMethod();
+ }
+ return copyAnswer.callRealMethod();
+ })
+ .when(client)
+ .copyBlob(Mockito.any(Path.class), Mockito.any(Path.class),
+ Mockito.nullable(String.class),
+ Mockito.any(TracingContext.class));
+
+ Mockito.doAnswer(deleteAnswer -> {
+ if (dirPathStr.equalsIgnoreCase(
+ ((Path) deleteAnswer.getArgument(0)).toUri().getPath())) {
+ Object result = deleteAnswer.callRealMethod();
+ tracingHeaderValidator.setOperatedBlobCount(null);
+ return result;
+ }
+ return deleteAnswer.callRealMethod();
+ })
+ .when(client)
+ .deleteBlobPath(Mockito.any(Path.class),
+ Mockito.nullable(String.class),
+ Mockito.any(TracingContext.class));
+
+ return answer.callRealMethod();
+ })
+ .when(store)
+ .rename(Mockito.any(Path.class), Mockito.any(Path.class),
+ Mockito.any(
+ RenameAtomicityUtils.class), Mockito.any(TracingContext.class));
+
+ fs.rename(new Path(dirPathStr), new Path("/dst/"));
+ }
+
+ /**
+ * Test to assert that the rename resume from FileStatus uses the same
+ * {@link TracingContext} object used by the initial ListStatus call.
+ * Also assert that last rename's copy and delete API call would push count
+ * of blobs operated in resume flow in clientRequestId.
+ */
+ @Test
+ public void testBlobRenameResumeWithListStatus() throws Exception {
+ assumeNonHnsAccountBlobEndpoint(getFileSystem());
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ store.setClient(client);
+
+ renameFailureSetup(fs, client, false);
+ AtomicInteger copied = assertTracingContextOnRenameResumeProcess(fs, store,
+ client, FSOperationType.LISTSTATUS);
+
+ fs.listStatus(new Path("/hbase"));
+ fs.registerListener(null);
+ Assertions.assertThat(fs.exists(new Path("/hbase/testDir2"))).isTrue();
+ Assertions.assertThat(copied.get()).isGreaterThan(0);
+ }
+
+ /**
+ * Test to assert that the rename resume from FileStatus uses the same
+ * {@link TracingContext} object used by the initial GetFileStatus call.
+ * Also assert that last rename's copy and delete API call would push count
+ * of blobs operated in resume flow in clientRequestId.
+ */
+ @Test
+ public void testBlobRenameResumeWithGetFileStatus() throws Exception {
+ assumeNonHnsAccountBlobEndpoint(getFileSystem());
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ store.setClient(client);
+
+ renameFailureSetup(fs, client, false);
+ AtomicInteger copied = assertTracingContextOnRenameResumeProcess(fs, store,
+ client, FSOperationType.GET_FILESTATUS);
+
+ intercept(FileNotFoundException.class, () -> {
+ fs.getFileStatus(new Path("/hbase/testDir"));
+ });
+ Assertions.assertThat(fs.exists(new Path("/hbase/testDir2"))).isTrue();
+ Assertions.assertThat(copied.get()).isGreaterThan(0);
+ }
+
+ private void renameFailureSetup(final AzureBlobFileSystem fs,
+ final AbfsClient client, final Boolean srcMarkerToBeDeleted)
+ throws Exception {
+ fs.mkdirs(new Path("/hbase/testDir"));
+ ExecutorService executorService = Executors.newFixedThreadPool(5);
+ List futures = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ final int iter = i;
+ futures.add(executorService.submit(
+ () -> fs.create(new Path("/hbase/testDir/file" + iter))));
+ }
+
+ for (Future future : futures) {
+ future.get();
+ }
+
+ if (srcMarkerToBeDeleted) {
+ client.deleteBlobPath(new Path("/hbase/testDir"), null,
+ getTestTracingContext(fs, true));
+ }
+
+ AbfsRestOperation op = client.acquireBlobLease("/hbase/testDir/file5", -1,
+ getTestTracingContext(fs, true));
+ String leaseId = op.getResult()
+ .getResponseHeader(HttpHeaderConfigurations.X_MS_LEASE_ID);
+
+ Map pathLeaseIdMap = new ConcurrentHashMap<>();
+ AtomicBoolean leaseCanBeTaken = new AtomicBoolean(true);
+ Mockito.doAnswer(answer -> {
+ if (!leaseCanBeTaken.get()) {
+ throw new RuntimeException();
+ }
+ AbfsRestOperation abfsRestOperation
+ = (AbfsRestOperation) answer.callRealMethod();
+ pathLeaseIdMap.put(answer.getArgument(0),
+ abfsRestOperation.getResult().getResponseHeader(X_MS_LEASE_ID));
+ return abfsRestOperation;
+ })
+ .when(client)
+ .acquireBlobLease(Mockito.anyString(), Mockito.anyInt(),
+ Mockito.any(TracingContext.class));
+
+ intercept(Exception.class, () -> {
+ fs.rename(new Path("/hbase/testDir"), new Path("/hbase/testDir2"));
+ });
+
+ leaseCanBeTaken.set(false);
+ for (Map.Entry entry : pathLeaseIdMap.entrySet()) {
+ try {
+ client.releaseBlobLease(entry.getKey(), entry.getValue(),
+ getTestTracingContext(fs, true));
+ } catch (Exception e) {
+
+ }
+ }
+ client.releaseBlobLease("/hbase/testDir/file5", leaseId,
+ getTestTracingContext(fs, true));
+ leaseCanBeTaken.set(true);
+ }
+
+ private AtomicInteger assertTracingContextOnRenameResumeProcess(final AzureBlobFileSystem fs,
+ final AzureBlobFileSystemStore store,
+ final AbfsClient client, final FSOperationType fsOperationType)
+ throws IOException {
+ final TracingHeaderValidator tracingHeaderValidator
+ = new TracingHeaderValidator(
+ fs.getAbfsStore().getAbfsConfiguration().getClientCorrelationId(),
+ fs.getFileSystemId(), fsOperationType, true, 0);
+ fs.registerListener(tracingHeaderValidator);
+
+ AtomicInteger copied = new AtomicInteger(0);
+ Mockito.doAnswer(answer -> {
+ copied.incrementAndGet();
+ Path path = answer.getArgument(0);
+ if ("/hbase/testDir".equalsIgnoreCase(path.toUri().getPath())) {
+ tracingHeaderValidator.setOperatedBlobCount(copied.get());
+ }
+ return answer.callRealMethod();
+ })
+ .when(store)
+ .copyBlob(Mockito.any(Path.class), Mockito.any(Path.class),
+ Mockito.nullable(String.class), Mockito.any(TracingContext.class));
+
+ /*
+ * RenameAtomicUtil internally calls Filesystem's API of read and delete
+ * which would have different primaryIds. But once renameAtomicUtil has read
+ * the RenamePending JSON, all the operations will use the same tracingContext
+ * which was started by ListStatus or GetFileStatus operation.
+ * This is the reason why the validation is disabled until the RenameAtomicUtil
+ * object reads the JSON.
+ * The filesystem's delete API called in RenameAtomicUtils.cleanup create a
+ * new TracingContext object with a new primaryRequestId and also updates the
+ * new primaryRequestId in the listener object of FileSystem. Therefore, once,
+ * cleanup method is completed, the listener is explicitly updated with the
+ * primaryRequestId it was using before the RenameAtomicUtils object was created.
+ */
+ Mockito.doAnswer(answer -> {
+ final String primaryRequestId = ((TracingContext) answer.getArgument(
+ 1)).getPrimaryRequestId();
+ tracingHeaderValidator.setDisableValidation(true);
+ RenameAtomicityUtils renameAtomicityUtils = Mockito.spy(
+ (RenameAtomicityUtils) answer.callRealMethod());
+ Mockito.doAnswer(cleanupAnswer -> {
+ tracingHeaderValidator.setDisableValidation(true);
+ cleanupAnswer.callRealMethod();
+ tracingHeaderValidator.setDisableValidation(false);
+ tracingHeaderValidator.updatePrimaryRequestID(primaryRequestId);
+ return null;
+ }).when(renameAtomicityUtils).cleanup(Mockito.any(Path.class));
+ tracingHeaderValidator.setDisableValidation(false);
+ return renameAtomicityUtils;
+ })
+ .when(fs)
+ .getRenameAtomicityUtilsForRedo(Mockito.any(Path.class),
+ Mockito.any(TracingContext.class), Mockito.anyString(), Mockito.any(
+ AbfsInputStream.class));
+
+ Mockito.doAnswer(answer -> {
+ answer.callRealMethod();
+ tracingHeaderValidator.setOperatedBlobCount(null);
+ return null;
+ })
+ .when(client)
+ .deleteBlobPath(Mockito.any(Path.class), Mockito.nullable(String.class),
+ Mockito.any(TracingContext.class));
+ return copied;
+ }
+
+ @Test
+ public void testProducerStopOnRenameFailure() throws Exception {
+ assumeNonHnsAccountBlobEndpoint(getFileSystem());
+ Configuration configuration = Mockito.spy(getRawConfiguration());
+ AzureBlobFileSystem fs = Mockito.spy(
+ (AzureBlobFileSystem) FileSystem.get(configuration));
+
+ fs.mkdirs(new Path("/src"));
+ ExecutorService executorService = Executors.newFixedThreadPool(10);
+ List futureList = new ArrayList<>();
+ for (int i = 0; i < 20; i++) {
+ int iter = i;
+ Future future = executorService.submit(() -> {
+ try {
+ fs.create(new Path("/src/file" + iter));
+ } catch (IOException ex) {}
+ });
+ futureList.add(future);
+ }
+
+ for (Future future : futureList) {
+ future.get();
+ }
+
+ AbfsClient client = fs.getAbfsClient();
+ AbfsClient spiedClient = Mockito.spy(client);
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ store.setClient(spiedClient);
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+
+ ListBlobProducer[] producers = new ListBlobProducer[1];
+ Mockito.doAnswer(answer -> {
+ producers[0] = (ListBlobProducer) answer.callRealMethod();
+ return producers[0];
+ }).when(store).getListBlobProducer(Mockito.anyString(), Mockito.any(
+ ListBlobQueue.class), Mockito.nullable(String.class),
+ Mockito.any(TracingContext.class));
+
+ AtomicInteger listCounter = new AtomicInteger(0);
+ AtomicBoolean hasConsumerStarted = new AtomicBoolean(false);
+
+ Mockito.doAnswer(answer -> {
+ String marker = answer.getArgument(0);
+ String prefix = answer.getArgument(1);
+ String delimiter = answer.getArgument(2);
+ TracingContext tracingContext = answer.getArgument(4);
+ int counter = listCounter.incrementAndGet();
+ if (counter > 1 && producers[0] != null) {
+ while (!hasConsumerStarted.get()) {
+ Thread.sleep(1_000L);
+ }
+ }
+ Object result = client.getListBlobs(marker, prefix, delimiter, 1,
+ tracingContext);
+ return result;
+ })
+ .when(spiedClient)
+ .getListBlobs(Mockito.nullable(String.class),
+ Mockito.nullable(String.class), Mockito.nullable(String.class),
+ Mockito.nullable(Integer.class), Mockito.any(TracingContext.class));
+
+ Mockito.doAnswer(answer -> {
+ spiedClient.acquireBlobLease(
+ ((Path) answer.getArgument(0)).toUri().getPath(), -1,
+ answer.getArgument(2));
+ hasConsumerStarted.set(true);
+ return answer.callRealMethod();
+ })
+ .when(spiedClient)
+ .deleteBlobPath(Mockito.any(Path.class), Mockito.nullable(String.class),
+ Mockito.any(TracingContext.class));
+
+ intercept(Exception.class, () -> {
+ fs.rename(new Path("/src"), new Path("/dst"));
+ });
+
+ producers[0].waitForProcessCompletion();
+
+ Mockito.verify(spiedClient, Mockito.atMost(3))
+ .getListBlobs(Mockito.nullable(String.class),
+ Mockito.nullable(String.class), Mockito.nullable(String.class),
+ Mockito.nullable(Integer.class), Mockito.any(TracingContext.class));
+ }
+
+ @Test
+ public void testRenameResumeThroughListStatusWithSrcDirDeleted() throws Exception {
+ assumeNonHnsAccountBlobEndpoint(getFileSystem());
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ store.setClient(client);
+
+ renameFailureSetup(fs, client, false);
+
+ fs.delete(new Path("/hbase/testDir"), true);
+ AtomicInteger copied = assertTracingContextOnRenameResumeProcess(fs, store,
+ client, FSOperationType.LISTSTATUS);
+ Assertions.assertThat(fs.listStatus(new Path("/hbase"))).hasSize(1);
+ Assertions.assertThat(copied.get()).isEqualTo(0);
+ }
+
+ @Test
+ public void testRenameResumeThroughListStatusWithSrcDirDeletedJustBeforeResume() throws Exception {
+ assumeNonHnsAccountBlobEndpoint(getFileSystem());
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ store.setClient(client);
+
+ renameFailureSetup(fs, client, false);
+
+ AtomicInteger copied = assertTracingContextOnRenameResumeProcess(fs, store,
+ client, FSOperationType.LISTSTATUS);
+ Mockito.doAnswer(answer -> {
+ String path = answer.getArgument(0);
+ if("/hbase/testDir".equalsIgnoreCase(path)) {
+ fs.delete(new Path(path), true);
+ }
+ return answer.callRealMethod();
+ }).when(client).acquireBlobLease(Mockito.anyString(), Mockito.anyInt(), Mockito.any(TracingContext.class));
+ Assertions.assertThat(fs.listStatus(new Path("/hbase"))).hasSize(1);
+ Assertions.assertThat(copied.get()).isEqualTo(0);
+ }
+
+ @Test
+ public void testRenameResumeThroughListStatusWhenSrcDirectoryETagIsChanged()
+ throws Exception {
+ assumeNonHnsAccountBlobEndpoint(getFileSystem());
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ store.setClient(client);
+
+ renameFailureSetup(fs, client, false);
+
+ fs.delete(new Path("/hbase/testDir"), true);
+ fs.mkdirs(new Path("/hbase/testDir"));
+ AtomicInteger copied = assertTracingContextOnRenameResumeProcess(fs, store,
+ client, FSOperationType.LISTSTATUS);
+ Assertions.assertThat(fs.listStatus(new Path("/hbase"))).hasSize(2);
+ Assertions.assertThat(copied.get()).isEqualTo(0);
+ }
+
+ @Test
+ public void testRenameResumeThroughGetStatusWhenSrcDirectoryETagIsChanged()
+ throws Exception {
+ assumeNonHnsAccountBlobEndpoint(getFileSystem());
+ AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ store.setClient(client);
+
+ renameFailureSetup(fs, client, false);
+
+ fs.delete(new Path("/hbase/testDir"), true);
+ fs.mkdirs(new Path("/hbase/testDir"));
+ AtomicInteger copied = assertTracingContextOnRenameResumeProcess(fs, store,
+ client, FSOperationType.GET_FILESTATUS);
+ fs.getFileStatus(new Path("/hbase/testDir"));
+ Assertions.assertThat(copied.get()).isEqualTo(0);
+ }
+
+ @Test
+ public void testRenameResumeThroughGetStatusWhenSrcDirMakerOnRenameCreatedOnDfsEndpoint()
+ throws Exception {
+ assumeNonHnsAccountBlobEndpoint(getFileSystem());
+ Configuration configuration = Mockito.spy(getRawConfiguration());
+ configuration.set(FS_AZURE_MKDIRS_FALLBACK_TO_DFS, "true");
+ commonTestRenameResumeThroughGetStatusWhenSrcDirMarkerCreatedOnRename(
+ configuration);
+ }
+
+ @Test
+ public void testRenameResumeThroughGetStatusWhenSrcDirMakerOnRenameCreatedOnBlobEndpoint()
+ throws Exception {
+ assumeNonHnsAccountBlobEndpoint(getFileSystem());
+ Configuration configuration = Mockito.spy(getRawConfiguration());
+ configuration.set(FS_AZURE_MKDIRS_FALLBACK_TO_DFS, "false");
+ commonTestRenameResumeThroughGetStatusWhenSrcDirMarkerCreatedOnRename(
+ configuration);
+ }
+
+ private void commonTestRenameResumeThroughGetStatusWhenSrcDirMarkerCreatedOnRename(
+ final Configuration configuration) throws Exception {
+ AzureBlobFileSystem fs = Mockito.spy(
+ (AzureBlobFileSystem) FileSystem.newInstance(
+ configuration));
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ store.setClient(client);
+
+ renameFailureSetup(fs, client, true);
+ intercept(FileNotFoundException.class, () -> {
+ fs.getFileStatus(new Path("/hbase/testDir"));
+ });
+ FileStatus[] fileStatuses = fs.listStatus(new Path("/hbase/testDir2"));
+ Assertions.assertThat(fileStatuses).hasSize(10);
+ }
+
+ @Test
+ public void testRenameResumeThroughListStatusWhenSrcDirMakerOnRenameCreatedOnDfsEndpoint()
+ throws Exception {
+ assumeNonHnsAccountBlobEndpoint(getFileSystem());
+ Configuration configuration = Mockito.spy(getRawConfiguration());
+ configuration.set(FS_AZURE_MKDIRS_FALLBACK_TO_DFS, "true");
+ commonTestRenameResumeThroughListStatusWhenSrcDirMarkerCreatedOnRename(
+ configuration);
+ }
+
+ @Test
+ public void testRenameResumeThroughListStatusWhenSrcDirMakerOnRenameCreatedOnBlobEndpoint()
+ throws Exception {
+ assumeNonHnsAccountBlobEndpoint(getFileSystem());
+ Configuration configuration = Mockito.spy(getRawConfiguration());
+ configuration.set(FS_AZURE_MKDIRS_FALLBACK_TO_DFS, "false");
+ commonTestRenameResumeThroughListStatusWhenSrcDirMarkerCreatedOnRename(
+ configuration);
+ }
+
+ private void commonTestRenameResumeThroughListStatusWhenSrcDirMarkerCreatedOnRename(
+ final Configuration configuration) throws Exception {
+ AzureBlobFileSystem fs = Mockito.spy(
+ (AzureBlobFileSystem) FileSystem.newInstance(
+ configuration));
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ Mockito.doReturn(store).when(fs).getAbfsStore();
+ store.setClient(client);
+
+ renameFailureSetup(fs, client, true);
+ Assertions.assertThat(fs.listStatus(new Path("/hbase/"))).hasSize(1);
+ FileStatus[] fileStatuses = fs.listStatus(new Path("/hbase/testDir2"));
+ Assertions.assertThat(fileStatuses).hasSize(10);
+ }
+
+ @Test
+ public void renameBlobDirParallelThreadToRenameOnDifferentTracingContext()
+ throws Exception {
+ Assume.assumeTrue(getPrefixMode(getFileSystem()) == PrefixMode.BLOB);
+ Configuration configuration = getRawConfiguration();
+ AzureBlobFileSystem fs = Mockito.spy(
+ (AzureBlobFileSystem) FileSystem.newInstance(configuration));
+ AzureBlobFileSystemStore spiedStore = Mockito.spy(fs.getAbfsStore());
+ AbfsClient spiedClient = Mockito.spy(fs.getAbfsClient());
+
+ Mockito.doReturn(spiedStore).when(fs).getAbfsStore();
+ spiedStore.setClient(spiedClient);
+
+ fs.mkdirs(new Path("/testDir"));
+ fs.create(new Path("/testDir/file1"));
+ fs.create(new Path("/testDir/file2"));
+
+ AbfsClientTestUtil.hookOnRestOpsForTracingContextSingularity(spiedClient);
+ fs.rename(new Path("/testDir"), new Path("/testDir2"));
+ }
}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java
index 044c325c8c8dc2..a4b668b5e48353 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java
@@ -20,12 +20,14 @@
import java.util.Arrays;
+import org.junit.Assume;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsDirectory;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsFile;
@@ -76,6 +78,8 @@ public ITestAzureBlobFileSystemRenameUnicode() throws Exception {
@Test
public void testRenameFileUsingUnicode() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
+ Assume.assumeTrue(fs.getAbfsStore().getAbfsConfiguration().getPrefixMode()
+ == PrefixMode.DFS || !destDir.contains(":"));
Path folderPath1 = new Path(srcDir);
assertMkdirs(fs, folderPath1);
assertIsDirectory(fs, folderPath1);
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java
index 2c0bd31bf8eeb8..8ef5f1d4516703 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java
@@ -202,7 +202,7 @@ public void testAppendWithCPK() throws Exception {
// Trying to append with correct CPK headers
AppendRequestParameters appendRequestParameters =
new AppendRequestParameters(
- 0, 0, 5, Mode.APPEND_MODE, false, null);
+ 0, 0, 5, Mode.APPEND_MODE, false, null, true);
byte[] buffer = getRandomBytesArray(5);
AbfsClient abfsClient = fs.getAbfsClient();
AbfsRestOperation abfsRestOperation = abfsClient
@@ -247,7 +247,7 @@ public void testAppendWithoutCPK() throws Exception {
// Trying to append without CPK headers
AppendRequestParameters appendRequestParameters =
new AppendRequestParameters(
- 0, 0, 5, Mode.APPEND_MODE, false, null);
+ 0, 0, 5, Mode.APPEND_MODE, false, null, true);
byte[] buffer = getRandomBytesArray(5);
AbfsClient abfsClient = fs.getAbfsClient();
AbfsRestOperation abfsRestOperation = abfsClient
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java
index 8b60dd801cb30e..58a2d99a336e34 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java
@@ -20,13 +20,25 @@
import java.net.URI;
+import org.assertj.core.api.Assertions;
import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
+
+import static org.apache.hadoop.fs.CommonPathCapabilities.ETAGS_AVAILABLE;
+import static org.apache.hadoop.fs.CommonPathCapabilities.ETAGS_PRESERVED_IN_RENAME;
+import static org.apache.hadoop.fs.CommonPathCapabilities.FS_ACLS;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_DNS_PREFIX;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.WASB_DNS_PREFIX;
+import static org.apache.hadoop.fs.azurebfs.constants.InternalConstants.CAPABILITY_SAFE_READAHEAD;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
/**
* Test AzureBlobFileSystem initialization.
@@ -39,11 +51,12 @@ public ITestFileSystemInitialization() throws Exception {
@Test
public void ensureAzureBlobFileSystemIsInitialized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
- final String accountName = getAccountName();
+ String accountName = getAccountName();
final String filesystem = getFileSystemName();
String scheme = this.getAuthType() == AuthType.SharedKey ? FileSystemUriSchemes.ABFS_SCHEME
: FileSystemUriSchemes.ABFS_SECURE_SCHEME;
+
assertEquals(fs.getUri(),
new URI(scheme,
filesystem + "@" + accountName,
@@ -55,8 +68,13 @@ public void ensureAzureBlobFileSystemIsInitialized() throws Exception {
@Test
public void ensureSecureAzureBlobFileSystemIsInitialized() throws Exception {
- final String accountName = getAccountName();
+ String accountName = getAccountName();
final String filesystem = getFileSystemName();
+ if (getFileSystem().getAbfsStore().getAbfsConfiguration().shouldEnableBlobEndPoint()) {
+ if (accountName.contains(ABFS_DNS_PREFIX)) {
+ accountName = accountName.replace(ABFS_DNS_PREFIX, WASB_DNS_PREFIX);
+ }
+ }
final URI defaultUri = new URI(FileSystemUriSchemes.ABFS_SECURE_SCHEME,
filesystem + "@" + accountName,
null,
@@ -74,4 +92,41 @@ public void ensureSecureAzureBlobFileSystemIsInitialized() throws Exception {
assertNotNull("working directory", fs.getWorkingDirectory());
}
}
+
+ @Test
+ public void testFileSystemCapabilities() throws Throwable {
+ final AzureBlobFileSystem fs = getFileSystem();
+
+ final Path p = new Path("}");
+ // etags always present
+ Assertions.assertThat(fs.hasPathCapability(p, ETAGS_AVAILABLE))
+ .describedAs("path capability %s in %s", ETAGS_AVAILABLE, fs)
+ .isTrue();
+ // readahead always correct
+ Assertions.assertThat(fs.hasPathCapability(p, CAPABILITY_SAFE_READAHEAD))
+ .describedAs("path capability %s in %s", CAPABILITY_SAFE_READAHEAD, fs)
+ .isTrue();
+
+ // etags-over-rename and ACLs are either both true or both false.
+ final boolean etagsAcrossRename = fs.hasPathCapability(p, ETAGS_PRESERVED_IN_RENAME);
+ final boolean acls = fs.hasPathCapability(p, FS_ACLS);
+ Assertions.assertThat(etagsAcrossRename)
+ .describedAs("capabilities %s=%s and %s=%s in %s",
+ ETAGS_PRESERVED_IN_RENAME, etagsAcrossRename,
+ FS_ACLS, acls, fs)
+ .isEqualTo(acls);
+ }
+
+ @Test
+ public void testCreateContainerOnFileSystemPath() throws Exception{
+ final AzureBlobFileSystem fs = getFileSystem();
+ // assert that createContainer fails for already existing fileSystem.
+ intercept(AbfsRestOperationException.class,
+ () -> fs.getAbfsStore().createFilesystem(getTestTracingContext(fs, true),
+ fs.getAbfsStore().getPrefixMode() == PrefixMode.BLOB));
+
+ fs.getAbfsStore().deleteFilesystem(getTestTracingContext(fs, true));
+ intercept(AbfsRestOperationException.class,
+ () -> fs.getAbfsStore().getFilesystemProperties(getTestTracingContext(fs, true)));
+ }
}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
index 01227691c31393..f42cd37879b11d 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
@@ -21,6 +21,7 @@
import java.io.IOException;
import java.util.UUID;
+import org.apache.hadoop.fs.azurebfs.services.OperativeEndpoint;
import org.junit.Assume;
import org.junit.Test;
import org.assertj.core.api.Assertions;
@@ -32,8 +33,11 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azurebfs.enums.Trilean;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_DNS_PREFIX;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.WASB_DNS_PREFIX;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doReturn;
@@ -58,8 +62,16 @@ public class ITestGetNameSpaceEnabled extends AbstractAbfsIntegrationTest {
private static final String FALSE_STR = "false";
private boolean isUsingXNSAccount;
+ private boolean useBlobEndpoint;
public ITestGetNameSpaceEnabled() throws Exception {
isUsingXNSAccount = getConfiguration().getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false);
+ super.setup();
+ AzureBlobFileSystemStore abfsStore = getAbfsStore(getFileSystem());
+ PrefixMode prefixMode = abfsStore.getPrefixMode();
+ AbfsConfiguration abfsConfiguration = abfsStore.getAbfsConfiguration();
+ useBlobEndpoint = !(OperativeEndpoint.isIngressEnabledOnDFS(prefixMode, abfsConfiguration) ||
+ OperativeEndpoint.isMkdirEnabledOnDFS(abfsConfiguration) ||
+ OperativeEndpoint.isReadEnabledOnDFS(abfsConfiguration));
}
@Test
@@ -118,12 +130,15 @@ private AzureBlobFileSystem getNewFSWithHnsConf(
rawConfig
.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, true);
rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
- getNonExistingUrl());
+ getNonExistingUrl(isNamespaceEnabledAccount));
return (AzureBlobFileSystem) FileSystem.get(rawConfig);
}
- private String getNonExistingUrl() {
+ private String getNonExistingUrl(String isNamespaceEnabled) {
String testUri = this.getTestUrl();
+ if (Boolean.parseBoolean(isNamespaceEnabled)) {
+ testUri = testUri.replace(WASB_DNS_PREFIX, ABFS_DNS_PREFIX);
+ }
return getAbfsScheme() + "://" + UUID.randomUUID() + testUri
.substring(testUri.indexOf("@"));
}
@@ -137,11 +152,20 @@ public void testFailedRequestWhenFSNotExist() throws Exception {
+ testUri.substring(testUri.indexOf("@"));
AzureBlobFileSystem fs = this.getFileSystem(nonExistingFsUrl);
- intercept(FileNotFoundException.class,
- "\"The specified filesystem does not exist.\", 404",
- ()-> {
- fs.getFileStatus(new Path("/")); // Run a dummy FS call
- });
+ if (useBlobEndpoint) {
+ intercept(FileNotFoundException.class,
+ "\"The specified container does not exist.\", 404",
+ ()-> {
+ fs.getFileStatus(new Path("/")); // Run a dummy FS call
+ });
+ }
+ else {
+ intercept(FileNotFoundException.class,
+ "\"The specified filesystem does not exist.\", 404",
+ ()-> {
+ fs.getFileStatus(new Path("/")); // Run a dummy FS call
+ });
+ }
}
@Test
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestListBlob.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestListBlob.java
new file mode 100644
index 00000000000000..72a652e0ce4f5c
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestListBlob.java
@@ -0,0 +1,200 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs;
+
+
+import java.io.IOException;
+import java.util.List;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Assume;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
+import org.apache.hadoop.fs.azurebfs.services.BlobProperty;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
+import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
+
+public class ITestListBlob extends
+ AbstractAbfsIntegrationTest {
+
+ public ITestListBlob() throws Exception {
+ super();
+ }
+
+ @Test
+ public void testListBlob() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ assumeNonHnsAccountBlobEndpoint(fs);
+ createBlob(fs, "/dir/");
+ List blobProperties;
+ /*
+ * Call getListBlob for a path with isDefinitiveDirSearch = false. Should give
+ * results including the directory blob(hdi_isfolder=true).
+ */
+ blobProperties = fs.getAbfsStore()
+ .getListBlobs(new Path("dir"), null, null,
+ getTestTracingContext(fs, true), null, false);
+ Assertions.assertThat(blobProperties)
+ .describedAs(
+ "BlobList should match the number of files created in tests + the directory itself")
+ .hasSize(11);
+
+ /*
+ * Call getListBlob for a path with isDefinitiveDirSearch = false. Should give
+ * results excluding the directory blob(hdi_isfolder=true).
+ */
+ blobProperties = fs.getAbfsStore()
+ .getListBlobs(new Path("dir"), null, null,
+ getTestTracingContext(fs, true), null, true);
+ Assertions.assertThat(blobProperties)
+ .describedAs(
+ "BlobList should match the number of files created in tests")
+ .hasSize(10);
+
+ /*
+ * Call getListBlob for a path with isDefinitiveDirSearch = false with
+ * maxResult more than the number of exact blobs. Should give results including
+ * the directory blob(hdi_isfolder=true).
+ */
+ blobProperties = fs.getAbfsStore()
+ .getListBlobs(new Path("dir"), null, null,
+ getTestTracingContext(fs, true), 13, false);
+ Assertions.assertThat(blobProperties)
+ .describedAs(
+ "BlobList should match the number of files created in tests + the directory itself")
+ .hasSize(11);
+
+ /*
+ * Call getListBlob for a path with isDefinitiveDirSearch = false with
+ * maxResult lesser than the number of exact blobs. Should give result size
+ * same as the maxResult
+ */
+ blobProperties = fs.getAbfsStore()
+ .getListBlobs(new Path("dir"), null, null,
+ getTestTracingContext(fs, true), 5, false);
+ Assertions.assertThat(blobProperties)
+ .describedAs(
+ "BlobList should match the number of maxResult given")
+ .hasSize(5);
+ }
+
+ @Test
+ public void testListBlobWithMarkers() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ assumeNonHnsAccountBlobEndpoint(fs);
+ createBlob(fs, "/dir/");
+ AbfsClient client = fs.getAbfsClient();
+ AbfsClient spiedClient = Mockito.spy(client);
+ fs.getAbfsStore().setClient(spiedClient);
+
+ /*
+ * Server can give lesser number of results. In this case, server will give
+ * nextMarker.
+ * In this case, server will return one object, expectation is that the client
+ * uses nextMarker to make calls for the remaining blobs.
+ */
+ int count[] = new int[1];
+ count[0] = 0;
+ Mockito.doAnswer(answer -> {
+ String marker = answer.getArgument(0);
+ String prefix = answer.getArgument(1);
+ String delimiter = answer.getArgument(2);
+ TracingContext tracingContext = answer.getArgument(4);
+ count[0]++;
+ return client.getListBlobs(marker, prefix, delimiter, 1, tracingContext);
+ }).when(spiedClient).getListBlobs(Mockito.nullable(String.class),
+ Mockito.anyString(), Mockito.anyString(), Mockito.nullable(Integer.class),
+ Mockito.any(TracingContext.class));
+
+ List blobProperties = fs.getAbfsStore()
+ .getListBlobs(new Path("dir"), null, null,
+ getTestTracingContext(fs, true), 5, false);
+ Assertions.assertThat(blobProperties)
+ .describedAs(
+ "BlobList should match the number of maxResult given")
+ .hasSize(5);
+ Assertions.assertThat(count[0])
+ .describedAs(
+ "Number of calls to backend should be equal to maxResult given")
+ .isEqualTo(5);
+ }
+
+ @Test
+ public void testListBlobWithDelimiter() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ fs.setWorkingDirectory(new Path("/"));
+ assumeNonHnsAccountBlobEndpoint(fs);
+ List blobProperties;
+
+ // Create three levels of hierarchy.
+ Path level0 = new Path("a");
+ Path level1 = new Path("a/b");
+ Path level2 = new Path("a/b/c");
+ fs.mkdirs(level0);
+ fs.mkdirs(level1);
+ fs.mkdirs(level2);
+
+ // Without delimiter, recursive listing will return all the children and sub children
+ // There will be no BlobPrefix element and only explicit blobs will be returned
+ blobProperties = fs.getAbfsStore()
+ .getListBlobs(level0.getParent(), null, null,
+ getTestTracingContext(fs, true), null, true);
+ Assertions.assertThat(blobProperties)
+ .describedAs(
+ "BlobList should return all blobs in hierarchy")
+ .hasSize(3);
+
+ // With delimiter, non-recursive listing will return only the immediate children
+ // There will be repetition of marker blobs.
+ blobProperties = fs.getAbfsStore()
+ .getListBlobs(level0.getParent(), null, "/",
+ getTestTracingContext(fs, true), null, true);
+ Assertions.assertThat(blobProperties)
+ .describedAs(
+ "BlobList should return only immediate Children")
+ .hasSize(2);
+
+ // ABFS Listing With delimiter, non-recursive listing will return only the immediate children
+ // There will be no repetition of marker blobs.
+ FileStatus[] fileStatuses = fs.listStatus(level0.getParent());
+ Assertions.assertThat(fileStatuses)
+ .describedAs(
+ "BlobList should return only immediate Children")
+ .hasSize(1);
+ }
+
+ private void assumeNonHnsAccountBlobEndpoint(final AzureBlobFileSystem fs) {
+ Assume.assumeTrue("To work on only on non-HNS Blob endpoint",
+ fs.getAbfsStore().getAbfsConfiguration().getPrefixMode()
+ == PrefixMode.BLOB);
+ }
+
+ private void createBlob(final AzureBlobFileSystem fs, final String pathString)
+ throws IOException {
+ int i = 0;
+ while (i < 10) {
+ fs.create(new Path(pathString + i));
+ i++;
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestListBlobProducer.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestListBlobProducer.java
new file mode 100644
index 00000000000000..eae5d43d4e696b
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestListBlobProducer.java
@@ -0,0 +1,242 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.constants.FSOperationType;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
+import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
+import org.apache.hadoop.fs.azurebfs.services.AbfsClientTestUtil;
+import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation;
+import org.apache.hadoop.fs.azurebfs.services.ListBlobConsumer;
+import org.apache.hadoop.fs.azurebfs.services.ListBlobProducer;
+import org.apache.hadoop.fs.azurebfs.services.ListBlobQueue;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
+import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
+
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_PRODUCER_QUEUE_MAX_SIZE;
+
+public class ITestListBlobProducer extends AbstractAbfsIntegrationTest {
+
+ public ITestListBlobProducer() throws Exception {
+ super();
+ }
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ Assume.assumeTrue(
+ getFileSystem().getAbfsStore().getAbfsConfiguration().getPrefixMode()
+ == PrefixMode.BLOB);
+ }
+
+ @Test
+ public void testProducerWaitingForConsumerLagToGoDown() throws Exception {
+ Configuration configuration = Mockito.spy(getRawConfiguration());
+ configuration.set(FS_AZURE_PRODUCER_QUEUE_MAX_SIZE, "10");
+ AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.newInstance(
+ configuration);
+ AbfsClient client = Mockito.spy(fs.getAbfsClient());
+ AbfsClient spiedClient = Mockito.spy(client);
+ fs.getAbfsStore().setClient(spiedClient);
+ fs.setWorkingDirectory(new Path("/"));
+ fs.mkdirs(new Path("/src"));
+ ExecutorService executor = Executors.newFixedThreadPool(5);
+ List futureList = new ArrayList<>();
+ for (int i = 0; i < 20; i++) {
+ int iter = i;
+ futureList.add(executor.submit(() -> {
+ return fs.create(new Path("/src/file" + iter));
+ }));
+ }
+ for(Future future : futureList) {
+ future.get();
+ }
+
+ AtomicInteger producedBlobs = new AtomicInteger(0);
+ AtomicInteger listBlobInvoked = new AtomicInteger(0);
+
+ final ITestListBlobProducer testObj = this;
+ final ListBlobQueue queue = new ListBlobQueue(
+ fs.getAbfsStore().getAbfsConfiguration().getProducerQueueMaxSize(),
+ 1);
+ final CountDownLatch latch = new CountDownLatch(10);
+
+ Mockito.doAnswer(answer -> {
+ synchronized (testObj) {
+ listBlobInvoked.incrementAndGet();
+ AbfsRestOperation op = client.getListBlobs(answer.getArgument(0),
+ answer.getArgument(1), answer.getArgument(2), 1, answer.getArgument(4));
+ producedBlobs.incrementAndGet();
+ latch.countDown();
+ if(producedBlobs.get() > 10) {
+ Assert.assertTrue(queue.availableSize() > 0);
+ }
+ return op;
+ }
+ })
+ .when(spiedClient)
+ .getListBlobs(Mockito.nullable(String.class),
+ Mockito.nullable(String.class), Mockito.nullable(String.class),
+ Mockito.nullable(Integer.class),
+ Mockito.nullable(TracingContext.class));
+
+ AbfsClientTestUtil.hookOnRestOpsForTracingContextSingularity(client);
+ TracingContext tracingContext = new TracingContext("clientCorrelationId",
+ "fileSystemId", FSOperationType.TEST_OP,
+ getConfiguration().getTracingHeaderFormat(),
+ null);
+ ListBlobProducer producer = new ListBlobProducer("src/", spiedClient, queue,
+ null, getTestTracingContext(fs, true));
+ ListBlobConsumer consumer = new ListBlobConsumer(queue);
+ latch.await();
+
+ int oldInvocation = listBlobInvoked.get();
+ Assert.assertTrue(listBlobInvoked.get() == oldInvocation);
+
+ while (!consumer.isCompleted()) {
+ synchronized (testObj) {
+ consumer.consume();
+ Assert.assertTrue(queue.availableSize() > 0);
+ }
+ }
+
+ Assert.assertTrue(producedBlobs.get() == 20);
+ }
+
+ @Test
+ public void testConsumerWhenProducerThrowException() throws Exception {
+ Configuration configuration = Mockito.spy(getRawConfiguration());
+ configuration.set(FS_AZURE_PRODUCER_QUEUE_MAX_SIZE, "10");
+ AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.newInstance(
+ configuration);
+ AbfsClient client = fs.getAbfsClient();
+ AbfsClient spiedClient = Mockito.spy(client);
+ fs.getAbfsStore().setClient(spiedClient);
+ fs.setWorkingDirectory(new Path("/"));
+ fs.mkdirs(new Path("/src"));
+ for (int i = 0; i < 20; i++) {
+ fs.create(new Path("/src/file" + i));
+ }
+
+ Mockito.doAnswer(answer -> {
+ throw new AbfsRestOperationException(HttpURLConnection.HTTP_CONFLICT, "",
+ "", new Exception(""));
+
+ })
+ .when(spiedClient)
+ .getListBlobs(Mockito.nullable(String.class),
+ Mockito.nullable(String.class), Mockito.nullable(String.class),
+ Mockito.nullable(Integer.class),
+ Mockito.nullable(TracingContext.class));
+
+ ListBlobQueue queue = new ListBlobQueue(getConfiguration().getProducerQueueMaxSize(),
+ getConfiguration().getProducerQueueMaxSize());
+ ListBlobProducer producer = new ListBlobProducer("src/", spiedClient, queue,
+ null, getTestTracingContext(fs, true));
+ ListBlobConsumer consumer = new ListBlobConsumer(queue);
+
+ Boolean exceptionCaught = false;
+ try {
+ while (!consumer.isCompleted()) {
+ consumer.consume();
+ }
+ } catch (AzureBlobFileSystemException e) {
+ exceptionCaught = true;
+ }
+
+ Assert.assertTrue(exceptionCaught);
+ }
+
+ @Test
+ public void testProducerStopOnConsumerFailure() throws Exception {
+ Configuration configuration = Mockito.spy(getRawConfiguration());
+ AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.newInstance(
+ configuration);
+
+ fs.setWorkingDirectory(new Path("/"));
+ fs.mkdirs(new Path("/src"));
+ ExecutorService executorService = Executors.newFixedThreadPool(10);
+ List futureList = new ArrayList<>();
+ for (int i = 0; i < 20; i++) {
+ int iter = i;
+ Future future = executorService.submit(() -> {
+ try {
+ fs.create(new Path("/src/file" + iter));
+ } catch (IOException ex) {}
+ });
+ futureList.add(future);
+ }
+
+ for (Future future : futureList) {
+ future.get();
+ }
+
+ AbfsClient client = fs.getAbfsClient();
+ AbfsClient spiedClient = Mockito.spy(client);
+ fs.getAbfsStore().setClient(spiedClient);
+
+ ListBlobQueue queue = new ListBlobQueue(
+ getConfiguration().getProducerQueueMaxSize(),
+ getConfiguration().getProducerQueueMaxSize());
+ ListBlobConsumer consumer = new ListBlobConsumer(queue);
+
+ Mockito.doAnswer(answer -> {
+ String marker = answer.getArgument(0);
+ String prefix = answer.getArgument(1);
+ String delimiter = answer.getArgument(2);
+ TracingContext tracingContext = answer.getArgument(4);
+ Object result = client.getListBlobs(marker, prefix, delimiter, 1,
+ tracingContext);
+ consumer.fail();
+ return result;
+ })
+ .when(spiedClient)
+ .getListBlobs(Mockito.nullable(String.class),
+ Mockito.nullable(String.class), Mockito.nullable(String.class),
+ Mockito.nullable(Integer.class), Mockito.any(TracingContext.class));
+
+ ListBlobProducer producer = new ListBlobProducer("src/", spiedClient, queue,
+ null, getTestTracingContext(fs, true));
+
+ producer.waitForProcessCompletion();
+ Mockito.verify(spiedClient, Mockito.times(1))
+ .getListBlobs(Mockito.nullable(String.class),
+ Mockito.nullable(String.class), Mockito.nullable(String.class),
+ Mockito.nullable(Integer.class), Mockito.any(TracingContext.class));
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestSharedKeyAuth.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestSharedKeyAuth.java
index fedddcc4b16fbf..250b463cd2ec5f 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestSharedKeyAuth.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestSharedKeyAuth.java
@@ -17,16 +17,20 @@
*/
package org.apache.hadoop.fs.azurebfs;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
import org.junit.Assume;
import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
-import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
+import java.io.IOException;
+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_DNS_PREFIX;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.WASB_DNS_PREFIX;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
public class ITestSharedKeyAuth extends AbstractAbfsIntegrationTest {
@@ -40,23 +44,23 @@ public void testWithWrongSharedKey() throws Exception {
Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
Configuration config = this.getRawConfiguration();
config.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION,
- false);
+ true);
+ AzureBlobFileSystemStore abfsStore = getAbfsStore(getFileSystem());
String accountName = this.getAccountName();
+
String configkKey = FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME + "." + accountName;
// a wrong sharedKey
String secret = "XjUjsGherkDpljuyThd7RpljhR6uhsFjhlxRpmhgD12lnj7lhfRn8kgPt5"
+ "+MJHS7UJNDER+jn6KP6Jnm2ONQlm==";
config.set(configkKey, secret);
- AbfsClient abfsClient = this.getFileSystem(config).getAbfsClient();
- intercept(AbfsRestOperationException.class,
- "\"Server failed to authenticate the request. Make sure the value of "
- + "Authorization header is formed correctly including the "
- + "signature.\", 403",
- () -> {
- abfsClient
- .getAclStatus("/", getTestTracingContext(getFileSystem(), false));
- });
+ intercept(IOException.class,
+ "\"Server failed to authenticate the request. Make sure the value of "
+ + "Authorization header is formed correctly including the "
+ + "signature.\", 403",
+ () -> {
+ FileSystem.newInstance(config);
+ });
}
}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestSmallWriteOptimization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestSmallWriteOptimization.java
index fce2b682f580a6..667defc4d2abaf 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestSmallWriteOptimization.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestSmallWriteOptimization.java
@@ -24,6 +24,8 @@
import java.util.Map;
import java.io.IOException;
+import org.apache.hadoop.fs.azurebfs.services.OperativeEndpoint;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
import org.assertj.core.api.Assertions;
import org.junit.Assume;
import org.junit.runners.Parameterized;
@@ -71,6 +73,7 @@ public class ITestSmallWriteOptimization extends AbstractAbfsScaleTest {
private static final int HALF_TEST_BUFFER_SIZE = TWO_MB / 2;
private static final int QUARTER_TEST_BUFFER_SIZE = TWO_MB / 4;
private static final int TEST_FLUSH_ITERATION = 2;
+ private PrefixMode prefixMode = PrefixMode.DFS;
@Parameterized.Parameter
public String testScenario;
@@ -305,13 +308,16 @@ public void testSmallWriteOptimization()
// Tests with Optimization should only run if service has the feature on by
// default. Default settings will be turned on when server support is
// available on all store prod regions.
- if (enableSmallWriteOptimization) {
- Assume.assumeTrue(serviceDefaultOptmSettings);
- }
final AzureBlobFileSystem currentfs = this.getFileSystem();
Configuration config = currentfs.getConf();
boolean isAppendBlobTestSettingEnabled = (config.get(FS_AZURE_TEST_APPENDBLOB_ENABLED) == "true");
+ prefixMode = currentfs.getAbfsStore().getPrefixMode();
+
+ if (enableSmallWriteOptimization) {
+ Assume.assumeTrue(serviceDefaultOptmSettings);
+ Assume.assumeTrue(prefixMode == PrefixMode.DFS);
+ }
// This optimization doesnt take effect when append blob is on.
Assume.assumeFalse(isAppendBlobTestSettingEnabled);
@@ -426,7 +432,9 @@ private void formulateSmallWriteTestAppendPattern(final AzureBlobFileSystem fs,
? 1 // 1 append (with flush and close param)
: (wasDataPendingToBeWrittenToServer)
? 2 // 1 append + 1 flush (with close)
- : 1); // 1 flush (with close)
+ : (recurringWriteSize == 0 && !OperativeEndpoint.isIngressEnabledOnDFS(getPrefixMode(fs), fs.getAbfsStore().getAbfsConfiguration()))
+ ? 0 // no flush or close on prefix mode blob
+ : 1); //1 flush (with close)
expectedTotalRequestsMade += totalAppendFlushCalls;
expectedRequestsMadeWithData += totalAppendFlushCalls;
@@ -445,10 +453,19 @@ private void formulateSmallWriteTestAppendPattern(final AzureBlobFileSystem fs,
testIteration--;
}
+ /**
+ * Above test iteration loop executes one of the below two patterns
+ * 1. Append + Close (triggers flush)
+ * 2. Append + Flush
+ * For both patters PutBlobList is complete in the iteration loop itself
+ * Hence with PrefixMode Blob, below close won't trigger any network call
+ */
opStream.close();
- expectedTotalRequestsMade += 1;
- expectedRequestsMadeWithData += 1;
+ if (OperativeEndpoint.isIngressEnabledOnDFS(getPrefixMode(fs), fs.getAbfsStore().getAbfsConfiguration())) {
+ expectedTotalRequestsMade += 1;
+ expectedRequestsMadeWithData += 1;
+ }
// no change in expectedBytesSent
assertOpStats(fs.getInstrumentationMap(), expectedTotalRequestsMade, expectedRequestsMadeWithData, expectedBytesSent);
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestTracingContext.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestTracingContext.java
index 006004850d0df5..1d0701125341e9 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestTracingContext.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestTracingContext.java
@@ -26,17 +26,20 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
import org.assertj.core.api.Assertions;
import org.junit.Assume;
import org.junit.AssumptionViolatedException;
import org.junit.Ignore;
import org.junit.Test;
+import org.mockito.Mockito;
import org.apache.hadoop.fs.CommonPathCapabilities;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants;
import org.apache.hadoop.fs.azurebfs.constants.FSOperationType;
import org.apache.hadoop.fs.azurebfs.enums.Trilean;
+import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation;
import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
@@ -106,9 +109,16 @@ public void checkCorrelationConfigValidation(String clientCorrelationId,
: null;
//request should not fail for invalid clientCorrelationID
- AbfsRestOperation op = fs.getAbfsClient()
- .createPath(path, false, true, permission, umask, false, null,
- tracingContext);
+ AbfsRestOperation op;
+ if (getPrefixMode(fs) == PrefixMode.DFS) {
+ op = fs.getAbfsClient()
+ .createPath(path, false, true, permission, umask, false, null,
+ tracingContext);
+ } else {
+ op = fs.getAbfsClient()
+ .createPathBlob(path, false, true, new HashMap(), null,
+ tracingContext);
+ }
int statusCode = op.getResult().getStatusCode();
Assertions.assertThat(statusCode).describedAs("Request should not fail")
@@ -118,7 +128,7 @@ public void checkCorrelationConfigValidation(String clientCorrelationId,
.replace("]", "");
Assertions.assertThat(requestHeader)
.describedAs("Client Request Header should match TracingContext")
- .isEqualTo(tracingContext.getHeader());
+ .isEqualTo(op.getLastTracingContext().getHeader());
}
@Ignore
@@ -130,10 +140,10 @@ public void runCorrelationTestForAllMethods() throws Exception {
testClasses.put(new ITestAzureBlobFileSystemListStatus(), //liststatus
ITestAzureBlobFileSystemListStatus.class.getMethod("testListPath"));
- testClasses.put(new ITestAbfsReadWriteAndSeek(MIN_BUFFER_SIZE), //open,
+ testClasses.put(new ITestAbfsReadWriteAndSeek(MIN_BUFFER_SIZE, true), //open,
// read, write
ITestAbfsReadWriteAndSeek.class.getMethod("testReadAheadRequestID"));
- testClasses.put(new ITestAbfsReadWriteAndSeek(MIN_BUFFER_SIZE), //read (bypassreadahead)
+ testClasses.put(new ITestAbfsReadWriteAndSeek(MIN_BUFFER_SIZE, false), //read (bypassreadahead)
ITestAbfsReadWriteAndSeek.class
.getMethod("testReadAndWriteWithDifferentBufferSizesAndSeek"));
testClasses.put(new ITestAzureBlobFileSystemAppend(), //append
@@ -198,4 +208,74 @@ public void testExternalOps() throws Exception {
fs.getAbfsStore().setNamespaceEnabled(Trilean.TRUE);
fs.access(new Path("/"), FsAction.READ);
}
+
+ @Test
+ public void testRetryPrimaryRequestIdWhenInitiallySuppliedEmpty() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final String fileSystemId = fs.getFileSystemId();
+ final String clientCorrelationId = fs.getClientCorrelationId();
+ final TracingHeaderFormat tracingHeaderFormat = TracingHeaderFormat.ALL_ID_FORMAT;
+ TracingContext tracingContext = new TracingContext(clientCorrelationId,
+ fileSystemId, FSOperationType.CREATE_FILESYSTEM, tracingHeaderFormat, new TracingHeaderValidator(
+ fs.getAbfsStore().getAbfsConfiguration().getClientCorrelationId(),
+ fs.getFileSystemId(), FSOperationType.CREATE_FILESYSTEM, false,
+ 0));
+ AbfsHttpOperation abfsHttpOperation = Mockito.mock(AbfsHttpOperation.class);
+ Mockito.doNothing().when(abfsHttpOperation).setRequestProperty(Mockito.anyString(), Mockito.anyString());
+ tracingContext.constructHeader(abfsHttpOperation, null);
+ String header = tracingContext.getHeader();
+ String clientRequestIdUsed = header.split(":")[1];
+ String[] clientRequestIdUsedParts = clientRequestIdUsed.split("-");
+ String assertionPrimaryId = clientRequestIdUsedParts[clientRequestIdUsedParts.length - 1];
+
+ tracingContext.setRetryCount(1);
+ tracingContext.setListener(new TracingHeaderValidator(
+ fs.getAbfsStore().getAbfsConfiguration().getClientCorrelationId(),
+ fs.getFileSystemId(), FSOperationType.CREATE_FILESYSTEM, false,
+ 1));
+
+ tracingContext.constructHeader(abfsHttpOperation, "RT");
+ header = tracingContext.getHeader();
+ String primaryRequestId = header.split(":")[3];
+
+ Assertions.assertThat(primaryRequestId)
+ .describedAs("PrimaryRequestId in a retried request's "
+ + "tracingContext should be equal to last part of original "
+ + "request's clientRequestId UUID")
+ .isEqualTo(assertionPrimaryId);
+ }
+
+ @Test
+ public void testRetryPrimaryRequestIdWhenInitiallySuppliedNonEmpty() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final String fileSystemId = fs.getFileSystemId();
+ final String clientCorrelationId = fs.getClientCorrelationId();
+ final TracingHeaderFormat tracingHeaderFormat = TracingHeaderFormat.ALL_ID_FORMAT;
+ TracingContext tracingContext = new TracingContext(clientCorrelationId,
+ fileSystemId, FSOperationType.CREATE_FILESYSTEM, tracingHeaderFormat, new TracingHeaderValidator(
+ fs.getAbfsStore().getAbfsConfiguration().getClientCorrelationId(),
+ fs.getFileSystemId(), FSOperationType.CREATE_FILESYSTEM, false,
+ 0));
+ tracingContext.setPrimaryRequestID();
+ AbfsHttpOperation abfsHttpOperation = Mockito.mock(AbfsHttpOperation.class);
+ Mockito.doNothing().when(abfsHttpOperation).setRequestProperty(Mockito.anyString(), Mockito.anyString());
+ tracingContext.constructHeader(abfsHttpOperation, null);
+ String header = tracingContext.getHeader();
+ String assertionPrimaryId = header.split(":")[3];
+
+ tracingContext.setRetryCount(1);
+ tracingContext.setListener(new TracingHeaderValidator(
+ fs.getAbfsStore().getAbfsConfiguration().getClientCorrelationId(),
+ fs.getFileSystemId(), FSOperationType.CREATE_FILESYSTEM, false,
+ 1));
+
+ tracingContext.constructHeader(abfsHttpOperation, "RT");
+ header = tracingContext.getHeader();
+ String primaryRequestId = header.split(":")[3];
+
+ Assertions.assertThat(primaryRequestId)
+ .describedAs("PrimaryRequestId in a retried request's tracingContext "
+ + "should be equal to PrimaryRequestId in the original request.")
+ .isEqualTo(assertionPrimaryId);
+ }
}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java
index 565eb38c4f70a7..9e40f22d231b05 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java
@@ -24,6 +24,9 @@
public final class TestConfigurationKeys {
public static final String FS_AZURE_ACCOUNT_NAME = "fs.azure.account.name";
public static final String FS_AZURE_ABFS_ACCOUNT_NAME = "fs.azure.abfs.account.name";
+ public static final String FS_AZURE_ABFS_ACCOUNT1_NAME = "fs.azure.abfs.account1.name";
+ public static final String FS_AZURE_ENABLE_AUTOTHROTTLING = "fs.azure.enable.autothrottling";
+ public static final String FS_AZURE_ANALYSIS_PERIOD = "fs.azure.analysis.period";
public static final String FS_AZURE_ACCOUNT_KEY = "fs.azure.account.key";
public static final String FS_AZURE_CONTRACT_TEST_URI = "fs.contract.test.fs.abfs";
public static final String FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT = "fs.azure.test.namespace.enabled";
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractEtag.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractEtag.java
index d498ae71a4b6f0..10e8302bdc6c33 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractEtag.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractEtag.java
@@ -21,6 +21,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractContractEtagTest;
import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.Assume;
/**
* Contract test for etag support.
@@ -38,6 +39,7 @@ public ITestAbfsFileSystemContractEtag() throws Exception {
public void setup() throws Exception {
binding.setup();
super.setup();
+ Assume.assumeTrue(binding.isNamespaceEnabled(binding.getFileSystem()));
// Base rename contract test class re-uses the test folder
// This leads to failures when the test is re-run as same ABFS test
// containers are re-used for test run and creation of source and
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientTestUtil.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientTestUtil.java
new file mode 100644
index 00000000000000..eba6746dcd36dc
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientTestUtil.java
@@ -0,0 +1,173 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.assertj.core.api.Assertions;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
+import org.apache.hadoop.util.functional.BiFunctionRaisingIOE;
+import org.apache.hadoop.util.functional.FunctionRaisingIOE;
+
+import static java.net.HttpURLConnection.HTTP_OK;
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_METHOD_GET;
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_METHOD_PUT;
+import static org.apache.hadoop.fs.azurebfs.services.AuthType.OAuth;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.nullable;
+
+public final class AbfsClientTestUtil {
+
+ private AbfsClientTestUtil() {
+
+ }
+
+ public static void setMockAbfsRestOperationForCopyBlobOperation(final AbfsClient spiedClient,
+ final BiFunctionRaisingIOE functionRaisingIOE) {
+
+ Mockito.doAnswer(answer -> {
+ final AbfsRestOperation spiedRestOp = Mockito.spy(new AbfsRestOperation(
+ AbfsRestOperationType.CopyBlob,
+ spiedClient,
+ HTTP_METHOD_PUT,
+ answer.getArgument(0),
+ answer.getArgument(1)
+ ));
+ final AbfsRestOperation actualCallMakerOp = new AbfsRestOperation(
+ AbfsRestOperationType.CopyBlob,
+ spiedClient,
+ HTTP_METHOD_PUT,
+ answer.getArgument(0),
+ answer.getArgument(1)
+ );
+ return functionRaisingIOE.apply(spiedRestOp, actualCallMakerOp);
+ })
+ .when(spiedClient)
+ .getCopyBlobOperation(Mockito.any(URL.class), Mockito.anyList());
+ }
+
+ public static void setMockAbfsRestOperationForListBlobOperation(
+ final AbfsClient spiedClient,
+ FunctionRaisingIOE functionRaisingIOE)
+ throws Exception {
+ ExponentialRetryPolicy retryPolicy = Mockito.mock(ExponentialRetryPolicy.class);
+ AbfsHttpOperation httpOperation = Mockito.mock(AbfsHttpOperation.class);
+ AbfsRestOperation abfsRestOperation = Mockito.spy(new AbfsRestOperation(
+ AbfsRestOperationType.GetListBlobProperties,
+ spiedClient,
+ HTTP_METHOD_GET,
+ null,
+ new ArrayList<>()
+ ));
+
+ Mockito.doReturn(abfsRestOperation).when(spiedClient).getListBlobOperation(any(), any());
+
+ addMockBehaviourToAbfsClient(spiedClient, retryPolicy);
+ addMockBehaviourToRestOpAndHttpOp(abfsRestOperation, httpOperation);
+
+ functionRaisingIOE.apply(httpOperation);
+ }
+
+ public static void addMockBehaviourToRestOpAndHttpOp(final AbfsRestOperation abfsRestOperation,
+ final AbfsHttpOperation httpOperation) throws IOException {
+ HttpURLConnection httpURLConnection = Mockito.mock(HttpURLConnection.class);
+ Mockito.doNothing().when(httpURLConnection)
+ .setRequestProperty(nullable(String.class), nullable(String.class));
+ Mockito.doReturn(httpURLConnection).when(httpOperation).getConnection();
+ Mockito.doReturn("").when(abfsRestOperation).getClientLatency();
+ Mockito.doReturn(httpOperation).when(abfsRestOperation).createHttpOperation();
+ }
+
+ public static void addMockBehaviourToAbfsClient(final AbfsClient abfsClient,
+ final ExponentialRetryPolicy retryPolicy) throws IOException {
+ Mockito.doReturn(OAuth).when(abfsClient).getAuthType();
+ Mockito.doReturn("").when(abfsClient).getAccessToken();
+ AbfsThrottlingIntercept intercept = Mockito.mock(
+ AbfsThrottlingIntercept.class);
+ Mockito.doReturn(intercept).when(abfsClient).getIntercept();
+ Mockito.doNothing()
+ .when(intercept)
+ .sendingRequest(any(), nullable(AbfsCounters.class));
+ Mockito.doNothing().when(intercept).updateMetrics(any(), any());
+
+ Mockito.doReturn(retryPolicy).when(abfsClient).getRetryPolicy();
+ Mockito.doReturn(true)
+ .when(retryPolicy)
+ .shouldRetry(nullable(Integer.class), nullable(Integer.class));
+ Mockito.doReturn(false).when(retryPolicy).shouldRetry(0, HTTP_OK);
+ Mockito.doReturn(false).when(retryPolicy).shouldRetry(1, HTTP_OK);
+ Mockito.doReturn(false).when(retryPolicy).shouldRetry(2, HTTP_OK);
+ }
+
+ public static void populateBlobListHelper(BlobList list, BlobProperty blob, String nextMarker) {
+ list.addBlobProperty(blob);
+ list.setNextMarker("nextMarker");
+ }
+
+ public static void hookOnRestOpsForTracingContextSingularity(AbfsClient client) {
+ Set tracingContextSet = new HashSet<>();
+ ReentrantLock lock = new ReentrantLock();
+ Answer answer = new Answer() {
+ @Override
+ public Object answer(final InvocationOnMock invocationOnMock)
+ throws Throwable {
+ AbfsRestOperation op = Mockito.spy((AbfsRestOperation) invocationOnMock.callRealMethod());
+ Mockito.doAnswer(completeExecuteInvocation -> {
+ lock.lock();
+ try {
+ TracingContext context = completeExecuteInvocation.getArgument(0);
+ Assertions.assertThat(tracingContextSet).doesNotContain(context);
+ tracingContextSet.add(context);
+ } finally {
+ lock.unlock();
+ }
+ return completeExecuteInvocation.callRealMethod();
+ }).when(op).completeExecute(Mockito.any(TracingContext.class));
+ return op;
+ }
+ };
+
+ Mockito.doAnswer(answer)
+ .when(client)
+ .getAbfsRestOperation(Mockito.any(AbfsRestOperationType.class),
+ Mockito.anyString(), Mockito.any(URL.class), Mockito.anyList(),
+ Mockito.nullable(byte[].class), Mockito.anyInt(), Mockito.anyInt(),
+ Mockito.nullable(String.class));
+ Mockito.doAnswer(answer)
+ .when(client)
+ .getAbfsRestOperation(Mockito.any(AbfsRestOperationType.class),
+ Mockito.anyString(), Mockito.any(URL.class), Mockito.anyList());
+ Mockito.doAnswer(answer)
+ .when(client)
+ .getAbfsRestOperation(Mockito.any(AbfsRestOperationType.class),
+ Mockito.anyString(), Mockito.any(URL.class), Mockito.anyList(),
+ Mockito.nullable(String.class));
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOpTestUtil.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOpTestUtil.java
new file mode 100644
index 00000000000000..575b674d7ab148
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOpTestUtil.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+
+public final class AbfsHttpOpTestUtil {
+ private AbfsHttpOpTestUtil() {
+
+ }
+
+ public static void setConnection(AbfsHttpOperation op, AbfsHttpOperation copyFrom) {
+ op.setConnection(copyFrom.getConnection());
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperationTestUtil.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperationTestUtil.java
new file mode 100644
index 00000000000000..fb11f6ae618354
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperationTestUtil.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+import java.io.IOException;
+
+import org.mockito.Mockito;
+
+import org.apache.hadoop.util.functional.BiFunctionRaisingIOE;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.times;
+
+public final class AbfsRestOperationTestUtil {
+
+ private AbfsRestOperationTestUtil() {
+
+ }
+
+ public static void addAbfsHttpOpProcessResponseMock(final AbfsRestOperation spiedRestOp,
+ BiFunctionRaisingIOE functionRaisingIOE)
+ throws IOException {
+ Mockito.doAnswer(answer -> {
+ AbfsHttpOperation op = new AbfsHttpOperation(spiedRestOp.getUrl(),
+ spiedRestOp.getMethod(), spiedRestOp.getRequestHeaders());
+ AbfsHttpOperation actualOp = new AbfsHttpOperation(spiedRestOp.getUrl(),
+ spiedRestOp.getMethod(), spiedRestOp.getRequestHeaders());
+ AbfsHttpOperation spiedOp = Mockito.spy(op);
+ return functionRaisingIOE.apply(spiedOp, actualOp);
+ }).when(spiedRestOp).createHttpOperation();
+ }
+
+ public static void setResult(final AbfsRestOperation op, final AbfsHttpOperation result) {
+ op.setResult(result);
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java
similarity index 58%
rename from hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java
rename to hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java
index a725bf3175a5c1..d98f58d236ab1c 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java
@@ -20,20 +20,45 @@
import java.io.IOException;
import java.lang.reflect.Field;
+import java.net.HttpURLConnection;
+import java.net.ProtocolException;
import java.net.URL;
import java.util.List;
+import java.util.Random;
import java.util.regex.Pattern;
+import org.junit.Ignore;
+import org.assertj.core.api.Assertions;
import org.junit.Test;
+import org.mockito.Mockito;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azurebfs.AbfsConfiguration;
+import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
+import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
+import org.apache.hadoop.fs.azurebfs.TestAbfsConfigurationFieldsValidation;
+import org.apache.hadoop.fs.azurebfs.constants.FSOperationType;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
+import org.apache.hadoop.fs.azurebfs.contracts.services.AppendRequestParameters;
import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys;
+import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
+import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderFormat;
import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
-import static org.assertj.core.api.Assertions.assertThat;
+import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.APPEND_ACTION;
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_METHOD_DELETE;
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_METHOD_PATCH;
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_METHOD_PUT;
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HUNDRED_CONTINUE;
+import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.EXPECT;
+import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_HTTP_METHOD_OVERRIDE;
+import static org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams.QUERY_PARAM_ACTION;
+import static org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams.QUERY_PARAM_POSITION;
+import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ABFS_ACCOUNT_NAME;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -59,14 +84,20 @@
* Test useragent of abfs client.
*
*/
-public final class TestAbfsClient {
+public final class ITestAbfsClient extends AbstractAbfsIntegrationTest {
private static final String ACCOUNT_NAME = "bogusAccountName.dfs.core.windows.net";
private static final String FS_AZURE_USER_AGENT_PREFIX = "Partner Service";
+ private static final String HUNDRED_CONTINUE_USER_AGENT = SINGLE_WHITE_SPACE + HUNDRED_CONTINUE + SEMICOLON;
+ private static final String TEST_PATH = "/testfile";
+ public static final int REDUCED_RETRY_COUNT = 2;
+ public static final int REDUCED_BACKOFF_INTERVAL = 100;
+ public static final int BUFFER_LENGTH = 5;
+ public static final int BUFFER_OFFSET = 0;
private final Pattern userAgentStringPattern;
- public TestAbfsClient(){
+ public ITestAbfsClient() throws Exception {
StringBuilder regEx = new StringBuilder();
regEx.append("^");
regEx.append(APN_VERSION);
@@ -124,7 +155,7 @@ public void verifybBasicInfo() throws Exception {
}
private void verifybBasicInfo(String userAgentStr) {
- assertThat(userAgentStr)
+ Assertions.assertThat(userAgentStr)
.describedAs("User-Agent string [" + userAgentStr
+ "] should be of the pattern: " + this.userAgentStringPattern.pattern())
.matches(this.userAgentStringPattern)
@@ -142,6 +173,7 @@ private void verifybBasicInfo(String userAgentStr) {
.contains(System.getProperty(OS_ARCH));
}
+ @Ignore
@Test
public void verifyUserAgentPrefix()
throws IOException, IllegalAccessException {
@@ -153,7 +185,7 @@ public void verifyUserAgentPrefix()
String userAgentStr = getUserAgentString(abfsConfiguration, false);
verifybBasicInfo(userAgentStr);
- assertThat(userAgentStr)
+ Assertions.assertThat(userAgentStr)
.describedAs("User-Agent string should contain " + FS_AZURE_USER_AGENT_PREFIX)
.contains(FS_AZURE_USER_AGENT_PREFIX);
@@ -163,11 +195,38 @@ public void verifyUserAgentPrefix()
userAgentStr = getUserAgentString(abfsConfiguration, false);
verifybBasicInfo(userAgentStr);
- assertThat(userAgentStr)
+ Assertions.assertThat(userAgentStr)
.describedAs("User-Agent string should not contain " + FS_AZURE_USER_AGENT_PREFIX)
.doesNotContain(FS_AZURE_USER_AGENT_PREFIX);
}
+ @Test
+ public void verifyUserAgentExpectHeader()
+ throws IOException, IllegalAccessException {
+ final Configuration configuration = new Configuration();
+ configuration.addResource(TEST_CONFIGURATION_FILE_NAME);
+ configuration.set(ConfigurationKeys.FS_AZURE_USER_AGENT_PREFIX_KEY, FS_AZURE_USER_AGENT_PREFIX);
+ configuration.setBoolean(ConfigurationKeys.FS_AZURE_ACCOUNT_IS_EXPECT_HEADER_ENABLED, true);
+ AbfsConfiguration abfsConfiguration = new AbfsConfiguration(configuration,
+ ACCOUNT_NAME);
+ String userAgentStr = getUserAgentString(abfsConfiguration, false);
+
+ verifybBasicInfo(userAgentStr);
+ Assertions.assertThat(userAgentStr)
+ .describedAs("User-Agent string should contain " + HUNDRED_CONTINUE_USER_AGENT)
+ .contains(HUNDRED_CONTINUE_USER_AGENT);
+
+ configuration.setBoolean(ConfigurationKeys.FS_AZURE_ACCOUNT_IS_EXPECT_HEADER_ENABLED, false);
+ abfsConfiguration = new AbfsConfiguration(configuration,
+ ACCOUNT_NAME);
+ userAgentStr = getUserAgentString(abfsConfiguration, false);
+
+ verifybBasicInfo(userAgentStr);
+ Assertions.assertThat(userAgentStr)
+ .describedAs("User-Agent string should not contain " + HUNDRED_CONTINUE_USER_AGENT)
+ .doesNotContain(HUNDRED_CONTINUE_USER_AGENT);
+ }
+
@Test
public void verifyUserAgentWithoutSSLProvider() throws Exception {
final Configuration configuration = new Configuration();
@@ -179,14 +238,14 @@ public void verifyUserAgentWithoutSSLProvider() throws Exception {
String userAgentStr = getUserAgentString(abfsConfiguration, true);
verifybBasicInfo(userAgentStr);
- assertThat(userAgentStr)
+ Assertions.assertThat(userAgentStr)
.describedAs("User-Agent string should contain sslProvider")
.contains(DelegatingSSLSocketFactory.getDefaultFactory().getProviderName());
userAgentStr = getUserAgentString(abfsConfiguration, false);
verifybBasicInfo(userAgentStr);
- assertThat(userAgentStr)
+ Assertions.assertThat(userAgentStr)
.describedAs("User-Agent string should not contain sslProvider")
.doesNotContain(DelegatingSSLSocketFactory.getDefaultFactory().getProviderName());
}
@@ -202,7 +261,7 @@ public void verifyUserAgentClusterName() throws Exception {
String userAgentStr = getUserAgentString(abfsConfiguration, false);
verifybBasicInfo(userAgentStr);
- assertThat(userAgentStr)
+ Assertions.assertThat(userAgentStr)
.describedAs("User-Agent string should contain cluster name")
.contains(clusterName);
@@ -212,7 +271,7 @@ public void verifyUserAgentClusterName() throws Exception {
userAgentStr = getUserAgentString(abfsConfiguration, false);
verifybBasicInfo(userAgentStr);
- assertThat(userAgentStr)
+ Assertions.assertThat(userAgentStr)
.describedAs("User-Agent string should not contain cluster name")
.doesNotContain(clusterName)
.describedAs("User-Agent string should contain UNKNOWN as cluster name config is absent")
@@ -230,7 +289,7 @@ public void verifyUserAgentClusterType() throws Exception {
String userAgentStr = getUserAgentString(abfsConfiguration, false);
verifybBasicInfo(userAgentStr);
- assertThat(userAgentStr)
+ Assertions.assertThat(userAgentStr)
.describedAs("User-Agent string should contain cluster type")
.contains(clusterType);
@@ -240,7 +299,7 @@ public void verifyUserAgentClusterType() throws Exception {
userAgentStr = getUserAgentString(abfsConfiguration, false);
verifybBasicInfo(userAgentStr);
- assertThat(userAgentStr)
+ Assertions.assertThat(userAgentStr)
.describedAs("User-Agent string should not contain cluster type")
.doesNotContain(clusterType)
.describedAs("User-Agent string should contain UNKNOWN as cluster type config is absent")
@@ -304,26 +363,34 @@ public static AbfsClient getMockAbfsClient(AbfsClient baseAbfsClientInstance,
when(client.createDefaultUriQueryBuilder()).thenCallRealMethod();
when(client.createRequestUrl(any(), any())).thenCallRealMethod();
when(client.getAccessToken()).thenCallRealMethod();
+ when(client.getAbfsRestOperation(
+ Mockito.any(AbfsRestOperationType.class), Mockito.anyString(),
+ Mockito.any(URL.class),
+ Mockito.anyList())).thenCallRealMethod();
when(client.getSharedKeyCredentials()).thenCallRealMethod();
when(client.createDefaultHeaders()).thenCallRealMethod();
-
+ when(client.getAbfsConfiguration()).thenReturn(abfsConfig);
+ when(client.getIntercept()).thenReturn(
+ AbfsThrottlingInterceptFactory.getInstance(
+ abfsConfig.getAccountName().substring(0,
+ abfsConfig.getAccountName().indexOf(DOT)), abfsConfig));
// override baseurl
- client = TestAbfsClient.setAbfsClientField(client, "abfsConfiguration",
+ client = ITestAbfsClient.setAbfsClientField(client, "abfsConfiguration",
abfsConfig);
// override baseurl
- client = TestAbfsClient.setAbfsClientField(client, "baseUrl",
+ client = ITestAbfsClient.setAbfsClientField(client, "baseUrl",
baseAbfsClientInstance.getBaseUrl());
// override auth provider
if (currentAuthType == AuthType.SharedKey) {
- client = TestAbfsClient.setAbfsClientField(client, "sharedKeyCredentials",
+ client = ITestAbfsClient.setAbfsClientField(client, "sharedKeyCredentials",
new SharedKeyCredentials(
abfsConfig.getAccountName().substring(0,
abfsConfig.getAccountName().indexOf(DOT)),
abfsConfig.getStorageAccountKey()));
} else {
- client = TestAbfsClient.setAbfsClientField(client, "tokenProvider",
+ client = ITestAbfsClient.setAbfsClientField(client, "tokenProvider",
abfsConfig.getTokenProvider());
}
@@ -331,7 +398,7 @@ public static AbfsClient getMockAbfsClient(AbfsClient baseAbfsClientInstance,
String userAgent = "APN/1.0 Azure Blob FS/3.4.0-SNAPSHOT (PrivateBuild "
+ "JavaJRE 1.8.0_252; Linux 5.3.0-59-generic/amd64; openssl-1.0; "
+ "UNKNOWN/UNKNOWN) MSFT";
- client = TestAbfsClient.setAbfsClientField(client, "userAgent", userAgent);
+ client = ITestAbfsClient.setAbfsClientField(client, "userAgent", userAgent);
return client;
}
@@ -395,4 +462,160 @@ public static AbfsRestOperation getRestOp(AbfsRestOperationType type,
url,
requestHeaders);
}
+
+ public static AccessTokenProvider getAccessTokenProvider(AbfsClient client) {
+ return client.getTokenProvider();
+ }
+
+ /**
+ * Test helper method to get random bytes array.
+ * @param length The length of byte buffer.
+ * @return byte buffer.
+ */
+ private byte[] getRandomBytesArray(int length) {
+ final byte[] b = new byte[length];
+ new Random().nextBytes(b);
+ return b;
+ }
+
+ /**
+ * Test to verify that client retries append request without
+ * expect header enabled if append with expect header enabled fails
+ * with 4xx kind of error.
+ * @throws Exception
+ */
+ @Test
+ public void testExpectHundredContinue() throws Exception {
+ // Get the filesystem.
+ final AzureBlobFileSystem fs = getFileSystem();
+
+ final Configuration configuration = new Configuration();
+ configuration.addResource(TEST_CONFIGURATION_FILE_NAME);
+ AbfsClient abfsClient = getClient(fs);
+
+ AbfsConfiguration abfsConfiguration = new AbfsConfiguration(configuration,
+ configuration.get(FS_AZURE_ABFS_ACCOUNT_NAME));
+
+ // Update the configuration with reduced retry count and reduced backoff interval.
+ AbfsConfiguration abfsConfig
+ = TestAbfsConfigurationFieldsValidation.updateRetryConfigs(
+ abfsConfiguration,
+ REDUCED_RETRY_COUNT, REDUCED_BACKOFF_INTERVAL);
+
+ // Gets the client.
+ AbfsClient testClient = Mockito.spy(
+ ITestAbfsClient.createTestClientFromCurrentContext(
+ abfsClient,
+ abfsConfig));
+
+ // Create the append request params with expect header enabled initially.
+ AppendRequestParameters appendRequestParameters
+ = new AppendRequestParameters(
+ BUFFER_OFFSET, BUFFER_OFFSET, BUFFER_LENGTH,
+ AppendRequestParameters.Mode.APPEND_MODE, false, null, true);
+
+ byte[] buffer = getRandomBytesArray(BUFFER_LENGTH);
+
+ // Create a test container to upload the data.
+ Path testPath = path(TEST_PATH);
+ fs.create(testPath);
+ String finalTestPath = testPath.toString()
+ .substring(testPath.toString().lastIndexOf("/"));
+
+ // Creates a list of request headers.
+ final List