Skip to content

HADOOP-17548. ABFS: Toggle Store Mkdirs request overwrite parameter #2729

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
surendralilhore merged 14 commits into from
Mar 14, 2021
Merged

HADOOP-17548. ABFS: Toggle Store Mkdirs request overwrite parameter #2729

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
9753df3
overwrite false
sumangala17 Feb 26, 2021
db867e8
no exception
sumangala17 Feb 27, 2021
7cd6fc7
cleanup
sumangala17 Feb 27, 2021
a0a7fac
pr
sumangala17 Feb 27, 2021
258f7aa
assert msg
sumangala17 Feb 27, 2021
76e1a86
fix mkdir filepath bug
sumangala17 Mar 1, 2021
b4c9c64
rm create diff
sumangala17 Mar 1, 2021
44f3b9c
import
sumangala17 Mar 1, 2021
cb7fe7c
Merge branch 'trunk' into HADOOP-17548
sumangala17 Mar 1, 2021
6bd6f09
revw
sumangala17 Mar 2, 2021
0c6c37c
test prod, hns
sumangala17 Mar 4, 2021
bd66b38
non-hns success
sumangala17 Mar 4, 2021
0c04022
nonhns bugfix + ovwrt false
sumangala17 Mar 4, 2021
1941387
nonhns true
sumangala17 Mar 5, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,11 @@ public class AbfsConfiguration{
DefaultValue = DEFAULT_FS_AZURE_ENABLE_CONDITIONAL_CREATE_OVERWRITE)
private boolean enableConditionalCreateOverwrite;

@BooleanConfigurationValidatorAnnotation(ConfigurationKey =
FS_AZURE_ENABLE_MKDIR_OVERWRITE, DefaultValue =
DEFAULT_FS_AZURE_ENABLE_MKDIR_OVERWRITE)
private boolean mkdirOverwrite;

@StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_APPEND_BLOB_KEY,
DefaultValue = DEFAULT_FS_AZURE_APPEND_BLOB_DIRECTORIES)
private String azureAppendBlobDirs;
Expand Down Expand Up @@ -621,6 +626,10 @@ public boolean isConditionalCreateOverwriteEnabled() {
return this.enableConditionalCreateOverwrite;
}

public boolean isEnabledMkdirOverwrite() {
return mkdirOverwrite;
}

public String getAppendBlobDirs() {
return this.azureAppendBlobDirs;
}
Expand Down
2 changes: 1 addition & 1 deletion ...p-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -473,7 +473,7 @@ public boolean mkdirs(final Path f, final FsPermission permission) throws IOExce
statIncrement(DIRECTORIES_CREATED);
return true;
} catch (AzureBlobFileSystemException ex) {
checkException(f, ex, AzureServiceErrorCode.PATH_ALREADY_EXISTS);
checkException(f, ex);
return true;
}
}
Expand Down
3 changes: 2 additions & 1 deletion ...ls/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -601,7 +601,8 @@ public void createDirectory(final Path path, final FsPermission permission, fina
umask,
isNamespaceEnabled);

final AbfsRestOperation op = client.createPath(getRelativePath(path), false, true,
final AbfsRestOperation op = client.createPath(getRelativePath(path),
false, abfsConfiguration.isEnabledMkdirOverwrite(),
isNamespaceEnabled ? getOctalNotation(permission) : null,
isNamespaceEnabled ? getOctalNotation(umask) : null, false, null);
perfInfo.registerResult(op.getResult()).registerSuccess(true);
Expand Down
1 change: 1 addition & 0 deletions ...hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ public final class ConfigurationKeys {
* overwritten only if there is a match on the eTag of existing file.
*/
public static final String FS_AZURE_ENABLE_CONDITIONAL_CREATE_OVERWRITE = "fs.azure.enable.conditional.create.overwrite";
public static final String FS_AZURE_ENABLE_MKDIR_OVERWRITE = "fs.azure.enable.mkdir.overwrite";
/** Provides a config to provide comma separated path prefixes on which Appendblob based files are created
* Default is empty. **/
public static final String FS_AZURE_APPEND_BLOB_KEY = "fs.azure.appendblob.directories";
Expand Down
1 change: 1 addition & 0 deletions ...azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ public final class FileSystemConfigurations {

public static final String DEFAULT_FS_AZURE_ATOMIC_RENAME_DIRECTORIES = "/hbase";
public static final boolean DEFAULT_FS_AZURE_ENABLE_CONDITIONAL_CREATE_OVERWRITE = true;
public static final boolean DEFAULT_FS_AZURE_ENABLE_MKDIR_OVERWRITE = true;
public static final String DEFAULT_FS_AZURE_APPEND_BLOB_DIRECTORIES = "";

public static final int DEFAULT_READ_AHEAD_QUEUE_DEPTH = -1;
Expand Down
1 change: 1 addition & 0 deletions ...azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpHeaderConfigurations.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ public final class HttpHeaderConfigurations {
public static final String USER_AGENT = "User-Agent";
public static final String X_HTTP_METHOD_OVERRIDE = "X-HTTP-Method-Override";
public static final String X_MS_CLIENT_REQUEST_ID = "x-ms-client-request-id";
public static final String X_MS_EXISTING_RESOURCE_TYPE = "x-ms-existing-resource-type";
public static final String X_MS_DATE = "x-ms-date";
public static final String X_MS_REQUEST_ID = "x-ms-request-id";
public static final String X_MS_VERSION = "x-ms-version";
Expand Down
11 changes: 10 additions & 1 deletion ...p-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,16 @@ public AbfsRestOperation createPath(final String path, final boolean isFile, fin
HTTP_METHOD_PUT,
url,
requestHeaders);
op.execute();
try {
op.execute();
} catch (AzureBlobFileSystemException ex) {
String existingResource =
Copy link
Contributor

@snvijaya snvijaya Mar 2, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

first check if the httpstatus code is 409 and (!isFile), then retrieve the existingResource header.

Copy link
Contributor Author

@sumangala-patki sumangala-patki Mar 2, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

op.getResult().getResponseHeader(X_MS_EXISTING_RESOURCE_TYPE);
if (!isFile && existingResource != null && existingResource.equals(DIRECTORY)) {
return op; //don't throw ex on mkdirs for existing directory
}
throw ex;
}
return op;
}

Expand Down
27 changes: 22 additions & 5 deletions ...doop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,14 @@
import org.junit.Test;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import static org.apache.hadoop.fs.contract.ContractTestUtils.assertMkdirs;

import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.CONNECTIONS_MADE;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_MKDIR_OVERWRITE;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertMkdirs;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;

/**
* Test mkdir operation.
Expand All @@ -41,10 +43,25 @@ public ITestAzureBlobFileSystemMkDir() throws Exception {

@Test
public void testCreateDirWithExistingDir() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
Configuration config = new Configuration(this.getRawConfiguration());
config.set(FS_AZURE_ENABLE_MKDIR_OVERWRITE,
Boolean.toString(false));
final AzureBlobFileSystem fs = getFileSystem(config);
Path path = new Path("testFolder");
assertMkdirs(fs, path);
assertMkdirs(fs, path);
assertMkdirs(fs, path); //checks that mkdirs returns true
long timeCreated = fs.getFileStatus(path).getModificationTime();
assertMkdirs(fs, path); //call to existing dir should return success
assertEquals("LMT should not be updated for existing dir", timeCreated,
fs.getFileStatus(path).getModificationTime());
}

@Test
public void createDirWithExistingFilename() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
Path path = new Path("testFilePath");
fs.create(path);
assertTrue(fs.getFileStatus(path).isFile());
intercept(FileAlreadyExistsException.class, () -> fs.mkdirs(path));
}

@Test
Expand Down