blobProperties = fs.getAbfsStore()
+ .getListBlobs(new Path("dir"), null,
+ Mockito.mock(TracingContext.class), null, false);
+ Assertions.assertThat(blobProperties)
+ .describedAs(
+ "BlobList should match the number of files created in tests + the directory itself")
+ .hasSize(11);
+
+ blobProperties = fs.getAbfsStore()
+ .getListBlobs(new Path("dir"), null,
+ Mockito.mock(TracingContext.class), null, true);
+ Assertions.assertThat(blobProperties)
+ .describedAs(
+ "BlobList should match the number of files created in tests")
+ .hasSize(10);
}
}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java
index 28d0883e21649..25cd33c4f64a1 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java
@@ -42,6 +42,7 @@
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode;
import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
@@ -189,6 +190,31 @@ public void testPosixRenameDirectory() throws Exception {
assertFalse(fs.exists(new Path("testDir2/test1/test2/test3")));
}
+ @Test
+ public void testRenameToRoot() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("/src1/src2"));
+ Assert.assertTrue(fs.rename(new Path("/src1/src2"), new Path("/")));
+ Assert.assertTrue(fs.exists(new Path("/src2")));
+ }
+
+ @Test(expected = IOException.class)
+ public void testRenameBlobToDstWithColonInPath() throws Exception{
+ AzureBlobFileSystem fs = getFileSystem();
+ assumeNonHnsAccountBlobEndpoint(fs);
+ fs.create(new Path("/src"));
+ fs.rename(new Path("/src"), new Path("/dst:file"));
+ }
+
+ @Test
+ public void testRenameBlobInSameDirectoryWithNoMarker() throws Exception {
+ AzureBlobFileSystem fs = getFileSystem();
+ assumeNonHnsAccountBlobEndpoint(fs);
+ fs.create(new Path("/srcDir/dir/file"));
+ fs.getAbfsStore().getClient().deleteBlobPath(new Path("/srcDir/dir"), Mockito.mock(TracingContext.class));
+ Assert.assertTrue(fs.rename(new Path("/srcDir/dir"), new Path("/srcDir")));
+ }
+
/**
*
* Test to check behaviour of rename API if the destination directory is already
@@ -1273,14 +1299,15 @@ public void testBlobRenameSrcDirHasNoMarker() throws Exception {
fs.getAbfsStore()
.getClient()
.deleteBlobPath(new Path("/test1"), Mockito.mock(TracingContext.class));
- Assert.assertNull(fs.getAbfsStore()
- .getBlobPropertyWithNotFoundHandling(new Path("/test1"),
- Mockito.mock(TracingContext.class)));
+ LambdaTestUtils.intercept(AbfsRestOperationException.class, () -> {
+ fs.getAbfsStore().getBlobProperty(new Path("/test1"),
+ Mockito.mock(TracingContext.class));
+ });
fs.mkdirs(new Path("/test2"));
fs.rename(new Path("/test1"), new Path("/test2"));
- Assert.assertNotNull(fs.getAbfsStore()
- .getBlobPropertyWithNotFoundHandling(new Path("/test2/test1"),
- Mockito.mock(TracingContext.class)));
+ Assert.assertTrue(fs.getAbfsStore()
+ .getBlobProperty(new Path("/test2/test1"),
+ Mockito.mock(TracingContext.class)).getIsDirectory());
}
@Test
@@ -1289,8 +1316,19 @@ public void testCopyBlobTakeTime() throws Exception {
assumeNonHnsAccountBlobEndpoint(fileSystem);
AzureBlobFileSystemStore store = Mockito.spy(fileSystem.getAbfsStore());
fileSystem.setAbfsStore(store);
- Mockito.doReturn(COPY_STATUS_PENDING).when(store)
- .getCopyBlobProgress(Mockito.any(AbfsRestOperation.class));
+ AbfsClient client = store.getClient();
+ AbfsClient spiedClient = Mockito.spy(client);
+ store.setClient(spiedClient);
+
+ Mockito.doAnswer(answer -> {
+ AbfsRestOperation op = Mockito.spy((AbfsRestOperation) answer.callRealMethod());
+ AbfsHttpOperation httpOp = Mockito.spy(op.getResult());
+ Mockito.doReturn(COPY_STATUS_PENDING).when(httpOp).getResponseHeader(
+ HttpHeaderConfigurations.X_MS_COPY_STATUS);
+ Mockito.doReturn(httpOp).when(op).getResult();
+ return op;
+ }).when(spiedClient).copyBlob(Mockito.any(Path.class), Mockito.any(Path.class),
+ Mockito.any(TracingContext.class));
fileSystem.create(new Path("/test1/file"));
fileSystem.rename(new Path("/test1/file"), new Path("/test1/file2"));
Assert.assertTrue(fileSystem.exists(new Path("/test1/file2")));
@@ -1305,10 +1343,28 @@ public void testCopyBlobTakeTimeAndEventuallyFail() throws Exception {
assumeNonHnsAccountBlobEndpoint(fileSystem);
AzureBlobFileSystemStore store = Mockito.spy(fileSystem.getAbfsStore());
fileSystem.setAbfsStore(store);
- Mockito.doReturn(COPY_STATUS_PENDING).when(store)
- .getCopyBlobProgress(Mockito.any(AbfsRestOperation.class));
- Mockito.doReturn(COPY_STATUS_FAILED).when(store).getCopyStatus(Mockito.any(
- AbfsHttpOperation.class));
+ AbfsClient client = store.getClient();
+ AbfsClient spiedClient = Mockito.spy(client);
+ store.setClient(spiedClient);
+
+ Mockito.doAnswer(answer -> {
+ AbfsRestOperation op = Mockito.spy((AbfsRestOperation) answer.callRealMethod());
+ AbfsHttpOperation httpOp = Mockito.spy(op.getResult());
+ Mockito.doReturn(COPY_STATUS_PENDING).when(httpOp).getResponseHeader(
+ HttpHeaderConfigurations.X_MS_COPY_STATUS);
+ Mockito.doReturn(httpOp).when(op).getResult();
+ return op;
+ }).when(spiedClient).copyBlob(Mockito.any(Path.class), Mockito.any(Path.class),
+ Mockito.any(TracingContext.class));
+ Mockito.doAnswer(answer -> {
+ AbfsRestOperation op = Mockito.spy((AbfsRestOperation) answer.callRealMethod());
+ AbfsHttpOperation httpOp = Mockito.spy(op.getResult());
+ Mockito.doReturn(COPY_STATUS_FAILED).when(httpOp).getResponseHeader(
+ HttpHeaderConfigurations.X_MS_COPY_STATUS);
+ Mockito.doReturn(httpOp).when(op).getResult();
+ return op;
+ }).when(spiedClient).getBlobProperty(Mockito.any(Path.class), Mockito.any(TracingContext.class));
+
fileSystem.create(new Path("/test1/file"));
Boolean copyBlobFailureCaught = false;
try {
@@ -1331,10 +1387,28 @@ public void testCopyBlobTakeTimeAndEventuallyAborted() throws Exception {
assumeNonHnsAccountBlobEndpoint(fileSystem);
AzureBlobFileSystemStore store = Mockito.spy(fileSystem.getAbfsStore());
fileSystem.setAbfsStore(store);
- Mockito.doReturn(COPY_STATUS_PENDING).when(store)
- .getCopyBlobProgress(Mockito.any(AbfsRestOperation.class));
- Mockito.doReturn(COPY_STATUS_ABORTED).when(store).getCopyStatus(Mockito.any(
- AbfsHttpOperation.class));
+ AbfsClient client = store.getClient();
+ AbfsClient spiedClient = Mockito.spy(client);
+ store.setClient(spiedClient);
+
+ Mockito.doAnswer(answer -> {
+ AbfsRestOperation op = Mockito.spy((AbfsRestOperation) answer.callRealMethod());
+ AbfsHttpOperation httpOp = Mockito.spy(op.getResult());
+ Mockito.doReturn(COPY_STATUS_PENDING).when(httpOp).getResponseHeader(
+ HttpHeaderConfigurations.X_MS_COPY_STATUS);
+ Mockito.doReturn(httpOp).when(op).getResult();
+ return op;
+ }).when(spiedClient).copyBlob(Mockito.any(Path.class), Mockito.any(Path.class),
+ Mockito.any(TracingContext.class));
+ Mockito.doAnswer(answer -> {
+ AbfsRestOperation op = Mockito.spy((AbfsRestOperation) answer.callRealMethod());
+ AbfsHttpOperation httpOp = Mockito.spy(op.getResult());
+ Mockito.doReturn(COPY_STATUS_ABORTED).when(httpOp).getResponseHeader(
+ HttpHeaderConfigurations.X_MS_COPY_STATUS);
+ Mockito.doReturn(httpOp).when(op).getResult();
+ return op;
+ }).when(spiedClient).getBlobProperty(Mockito.any(Path.class), Mockito.any(TracingContext.class));
+
fileSystem.create(new Path("/test1/file"));
Boolean copyBlobFailureCaught = false;
try {
@@ -1359,11 +1433,21 @@ public void testCopyBlobTakeTimeAndBlobIsDeleted() throws Exception {
String srcFile = "/test1/file";
String dstFile = "/test1/file2";
fileSystem.setAbfsStore(store);
+ AbfsClient client = store.getClient();
+ AbfsClient spiedClient = Mockito.spy(client);
+ store.setClient(spiedClient);
+
Mockito.doAnswer(answer -> {
- fileSystem.delete(new Path(dstFile), false);
- return COPY_STATUS_PENDING;
- }).when(store)
- .getCopyBlobProgress(Mockito.any(AbfsRestOperation.class));
+ AbfsRestOperation op = Mockito.spy((AbfsRestOperation) answer.callRealMethod());
+ fileSystem.delete(new Path(dstFile), false);
+ AbfsHttpOperation httpOp = Mockito.spy(op.getResult());
+ Mockito.doReturn(COPY_STATUS_PENDING).when(httpOp).getResponseHeader(
+ HttpHeaderConfigurations.X_MS_COPY_STATUS);
+ Mockito.doReturn(httpOp).when(op).getResult();
+ return op;
+ }).when(spiedClient).copyBlob(Mockito.any(Path.class), Mockito.any(Path.class),
+ Mockito.any(TracingContext.class));
+
fileSystem.create(new Path(srcFile));
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java
index 044c325c8c8dc..a4b668b5e4835 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java
@@ -20,12 +20,14 @@
import java.util.Arrays;
+import org.junit.Assume;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsDirectory;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsFile;
@@ -76,6 +78,8 @@ public ITestAzureBlobFileSystemRenameUnicode() throws Exception {
@Test
public void testRenameFileUsingUnicode() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
+ Assume.assumeTrue(fs.getAbfsStore().getAbfsConfiguration().getPrefixMode()
+ == PrefixMode.DFS || !destDir.contains(":"));
Path folderPath1 = new Path(srcDir);
assertMkdirs(fs, folderPath1);
assertIsDirectory(fs, folderPath1);
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestListBlob.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestListBlob.java
index d8d4da35ecb01..4ae0a6df29c1e 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestListBlob.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestListBlob.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.fs.azurebfs;
+import java.io.IOException;
import java.util.List;
import org.assertj.core.api.Assertions;
@@ -28,6 +29,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
+import org.apache.hadoop.fs.azurebfs.services.BlobProperty;
import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
@@ -42,102 +44,97 @@ public ITestListBlob() throws Exception {
public void testListBlob() throws Exception {
AzureBlobFileSystem fs = getFileSystem();
assumeNonHnsAccountBlobEndpoint(fs);
- int i = 0;
- while (i < 10) {
- fs.create(new Path("/dir/" + i));
- i++;
- }
- List blobProperties = fs.getAbfsStore()
- .getListBlobs(new Path("dir"),
- Mockito.mock(TracingContext.class), null, null, false);
+ createBlob(fs, "/dir/");
+ List blobProperties;
+ /*
+ * Call getListBlob for a path with isDefinitiveDirSearch = false. Should give
+ * results including the directory blob(hdi_isfolder=true).
+ */
+ blobProperties = fs.getAbfsStore()
+ .getListBlobs(new Path("dir"), null,
+ Mockito.mock(TracingContext.class), null, false);
Assertions.assertThat(blobProperties)
.describedAs(
"BlobList should match the number of files created in tests + the directory itself")
.hasSize(11);
+ /*
+ * Call getListBlob for a path with isDefinitiveDirSearch = false. Should give
+ * results excluding the directory blob(hdi_isfolder=true).
+ */
blobProperties = fs.getAbfsStore()
- .getListBlobs(new Path("dir"),
- Mockito.mock(TracingContext.class), null, null, true);
+ .getListBlobs(new Path("dir"), null,
+ Mockito.mock(TracingContext.class), null, true);
Assertions.assertThat(blobProperties)
.describedAs(
"BlobList should match the number of files created in tests")
.hasSize(10);
- }
- @Test
- public void testListBlobWithMarkers() throws Exception {
- AzureBlobFileSystem fs = getFileSystem();
- assumeNonHnsAccountBlobEndpoint(fs);
- int i = 0;
- while (i < 10) {
- fs.create(new Path("/dir/" + i));
- i++;
- }
- AbfsClient spiedClient = Mockito.spy(fs.getAbfsClient());
- fs.getAbfsStore().setClient(spiedClient);
- List blobProperties = fs.getAbfsStore()
- .getListBlobs(new Path("dir"),
- Mockito.mock(TracingContext.class), 1, null, false);
+ /*
+ * Call getListBlob for a path with isDefinitiveDirSearch = false with
+ * maxResult more than the number of exact blobs. Should give results including
+ * the directory blob(hdi_isfolder=true).
+ */
+ blobProperties = fs.getAbfsStore()
+ .getListBlobs(new Path("dir"), null,
+ Mockito.mock(TracingContext.class), 13, false);
Assertions.assertThat(blobProperties)
.describedAs(
"BlobList should match the number of files created in tests + the directory itself")
.hasSize(11);
- Mockito.verify(spiedClient, Mockito.times(11))
- .getListBlobs(Mockito.any(Path.class),
- Mockito.any(TracingContext.class),
- Mockito.nullable(String.class), Mockito.nullable(String.class),
- Mockito.anyInt(), Mockito.anyBoolean());
+ /*
+ * Call getListBlob for a path with isDefinitiveDirSearch = false with
+ * maxResult lesser than the number of exact blobs. Should give result size
+ * same as the maxResult
+ */
blobProperties = fs.getAbfsStore()
- .getListBlobs(new Path("dir"),
- Mockito.mock(TracingContext.class), 1, null, true);
+ .getListBlobs(new Path("dir"), null,
+ Mockito.mock(TracingContext.class), 5, false);
Assertions.assertThat(blobProperties)
.describedAs(
- "BlobList should match the number of files created in tests + the directory itself")
- .hasSize(10);
- Mockito.verify(spiedClient, Mockito.times(21))
- .getListBlobs(Mockito.any(Path.class),
- Mockito.any(TracingContext.class),
- Mockito.nullable(String.class), Mockito.nullable(String.class),
- Mockito.anyInt(), Mockito.anyBoolean());
+ "BlobList should match the number of maxResult given")
+ .hasSize(5);
}
@Test
- public void testListBlobWithMarkersWithMaxResult() throws Exception {
+ public void testListBlobWithMarkers() throws Exception {
AzureBlobFileSystem fs = getFileSystem();
assumeNonHnsAccountBlobEndpoint(fs);
- int i = 0;
- while (i < 10) {
- fs.create(new Path("/dir/" + i));
- i++;
- }
- AbfsClient spiedClient = Mockito.spy(fs.getAbfsClient());
+ createBlob(fs, "/dir/");
+ AbfsClient client = fs.getAbfsClient();
+ AbfsClient spiedClient = Mockito.spy(client);
fs.getAbfsStore().setClient(spiedClient);
+
+ /*
+ * Server can give lesser number of results. In this case, server will give
+ * nextMarker.
+ * In this case, server will return one object, expectation is that the client
+ * uses nextMarker to make calls for the remaining blobs.
+ */
+ int count[] = new int[1];
+ count[0] = 0;
+ Mockito.doAnswer(answer -> {
+ String marker = answer.getArgument(0);
+ String prefix = answer.getArgument(1);
+ TracingContext tracingContext = answer.getArgument(3);
+ count[0]++;
+ return client.getListBlobs(marker, prefix, 1, tracingContext);
+ }).when(spiedClient).getListBlobs(Mockito.nullable(String.class),
+ Mockito.anyString(), Mockito.nullable(Integer.class),
+ Mockito.any(TracingContext.class));
+
List blobProperties = fs.getAbfsStore()
- .getListBlobs(new Path("dir"),
- Mockito.mock(TracingContext.class), 1, 5, false);
+ .getListBlobs(new Path("dir"), null,
+ Mockito.mock(TracingContext.class), 5, false);
Assertions.assertThat(blobProperties)
.describedAs(
- "BlobList should match the number of files created in tests + the directory itself")
+ "BlobList should match the number of maxResult given")
.hasSize(5);
- Mockito.verify(spiedClient, Mockito.times(5))
- .getListBlobs(Mockito.any(Path.class),
- Mockito.any(TracingContext.class),
- Mockito.nullable(String.class), Mockito.nullable(String.class),
- Mockito.anyInt(), Mockito.anyBoolean());
-
- blobProperties = fs.getAbfsStore()
- .getListBlobs(new Path("dir"),
- Mockito.mock(TracingContext.class), 1, 5, true);
- Assertions.assertThat(blobProperties)
+ Assertions.assertThat(count[0])
.describedAs(
- "BlobList should match the number of files created in tests + the directory itself")
- .hasSize(5);
- Mockito.verify(spiedClient, Mockito.times(10))
- .getListBlobs(Mockito.any(Path.class),
- Mockito.any(TracingContext.class),
- Mockito.nullable(String.class), Mockito.nullable(String.class),
- Mockito.anyInt(), Mockito.anyBoolean());
+ "Number of calls to backend should be equal to maxResult given")
+ .isEqualTo(5);
}
private void assumeNonHnsAccountBlobEndpoint(final AzureBlobFileSystem fs) {
@@ -145,4 +142,13 @@ private void assumeNonHnsAccountBlobEndpoint(final AzureBlobFileSystem fs) {
fs.getAbfsStore().getAbfsConfiguration().getPrefixMode()
== PrefixMode.BLOB);
}
+
+ private void createBlob(final AzureBlobFileSystem fs, final String pathString)
+ throws IOException {
+ int i = 0;
+ while (i < 10) {
+ fs.create(new Path(pathString + i));
+ i++;
+ }
+ }
}