diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 9087ccfe17ec5..f906b443f9b1f 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -889,8 +889,7 @@ public AccessTokenProvider getTokenProvider() throws TokenAccessProviderExceptio public SASTokenProvider getSASTokenProvider() throws AzureBlobFileSystemException { AuthType authType = getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); if (authType != AuthType.SAS) { - throw new SASTokenProviderException(String.format( - "Invalid auth type: %s is being used, expecting SAS", authType)); + throw new SASTokenProviderException(String.format("Invalid auth type: %s is being used, expecting SAS", authType)); } try { @@ -917,10 +916,8 @@ else if (sasTokenProviderClass != null) { finalSasTokenProviderClass = FixedSASTokenProvider.class; } - SASTokenProvider sasTokenProvider = ReflectionUtils - .newInstance(finalSasTokenProviderClass, rawConfig); - Preconditions.checkArgument(sasTokenProvider != null, - String.format("Failed to initialize %s", finalSasTokenProviderClass)); + SASTokenProvider sasTokenProvider = ReflectionUtils.newInstance(finalSasTokenProviderClass, rawConfig); + Preconditions.checkArgument(sasTokenProvider != null, String.format("Failed to initialize %s", finalSasTokenProviderClass)); LOG.trace("Initializing {}", finalSasTokenProviderClass.getName()); sasTokenProvider.initialize(rawConfig, accountName); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index f94fcb02160ac..a5b09d6abd4eb 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -193,6 +193,7 @@ public void initialize(URI uri, Configuration configuration) this.abfsStore = new AzureBlobFileSystemStore(systemStoreBuilder); LOG.trace("AzureBlobFileSystemStore init complete"); + final AbfsConfiguration abfsConfiguration = abfsStore .getAbfsConfiguration(); clientCorrelationId = TracingContext.validateClientCorrelationID( @@ -203,7 +204,6 @@ public void initialize(URI uri, Configuration configuration) if (abfsConfiguration.getCreateRemoteFileSystemDuringInitialization()) { TracingContext tracingContext = new TracingContext(clientCorrelationId, fileSystemId, FSOperationType.CREATE_FILESYSTEM, tracingHeaderFormat, listener); - if (this.tryGetFileStatus(new Path(AbfsHttpConstants.ROOT_PATH), tracingContext) == null) { try { this.createFileSystem(tracingContext); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 752d7b1a29eb4..4a1537eeaf88a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -55,8 +55,6 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.fs.azurebfs.constants.*; -import org.apache.hadoop.fs.azurebfs.utils.*; import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.base.Strings; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures; @@ -71,6 +69,10 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants; +import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes; +import org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations; +import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.ConcurrentWriteOperationDetectedException; @@ -112,6 +114,11 @@ import org.apache.hadoop.fs.azurebfs.services.AbfsPerfTracker; import org.apache.hadoop.fs.azurebfs.services.AbfsPerfInfo; import org.apache.hadoop.fs.azurebfs.services.ListingSupport; +import org.apache.hadoop.fs.azurebfs.utils.Base64; +import org.apache.hadoop.fs.azurebfs.utils.CRC64; +import org.apache.hadoop.fs.azurebfs.utils.DateTimeUtils; +import org.apache.hadoop.fs.azurebfs.utils.TracingContext; +import org.apache.hadoop.fs.azurebfs.utils.UriUtils; import org.apache.hadoop.fs.impl.OpenFileParameters; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; @@ -232,7 +239,6 @@ public AzureBlobFileSystemStore( boolean useHttps = (usingOauth || abfsConfiguration.isHttpsAlwaysUsed()) ? true : abfsStoreBuilder.isSecureScheme; this.abfsPerfTracker = new AbfsPerfTracker(fileSystemName, accountName, this.abfsConfiguration); this.abfsCounters = abfsStoreBuilder.abfsCounters; - initializeClient(uri, fileSystemName, accountName, useHttps); final Class identityTransformerClass = abfsStoreBuilder.configuration.getClass(FS_AZURE_IDENTITY_TRANSFORM_CLASS, IdentityTransformer.class, @@ -1602,13 +1608,14 @@ private void initializeClient(URI uri, String fileSystemName, abfsConfiguration.getStorageAccountKey()); } else if (authType == AuthType.SAS) { LOG.trace("Fetching SAS Token Provider"); - sasTokenProvider = this.abfsConfiguration.getSASTokenProvider(); + sasTokenProvider = abfsConfiguration.getSASTokenProvider(); } else { LOG.trace("Fetching token provider"); tokenProvider = abfsConfiguration.getTokenProvider(); ExtensionHelper.bind(tokenProvider, uri, abfsConfiguration.getRawConfiguration()); } + LOG.trace("Initializing AbfsClient for {}", baseUrl); if (tokenProvider != null) { this.client = new AbfsClient(baseUrl, creds, abfsConfiguration, diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/FixedSASTokenProvider.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/FixedSASTokenProvider.java index 7d4be92e5c6f8..786f466f64402 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/FixedSASTokenProvider.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/FixedSASTokenProvider.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.fs.azurebfs.extensions; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 33a6e78344ae2..3acbfc06a3f67 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -1127,8 +1127,7 @@ private String appendSASTokenToQuery(String path, try { LOG.trace("Fetch SAS token for {} on {}", operation, path); if (cachedSasToken == null) { - sasToken = sasTokenProvider.getSASToken(this.accountName, - this.filesystem, path, operation); + sasToken = sasTokenProvider.getSASToken(this.accountName, this.filesystem, path, operation); if ((sasToken == null) || sasToken.isEmpty()) { throw new UnsupportedOperationException("SASToken received is empty or null"); }