diff --git a/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GcsAccessTokenProvider.java b/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GcsAccessTokenProvider.java index 215e6c0d4b28..21b4877ff704 100644 --- a/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GcsAccessTokenProvider.java +++ b/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GcsAccessTokenProvider.java @@ -16,6 +16,8 @@ import com.google.cloud.hadoop.util.AccessTokenProvider; import org.apache.hadoop.conf.Configuration; +import java.time.Instant; + import static com.google.common.base.Strings.nullToEmpty; import static java.util.concurrent.TimeUnit.HOURS; @@ -29,7 +31,7 @@ public class GcsAccessTokenProvider @Override public AccessToken getAccessToken() { - return new AccessToken(nullToEmpty(config.get(GCS_ACCESS_TOKEN_CONF)), EXPIRATION_TIME_MILLISECONDS); + return new AccessToken(nullToEmpty(config.get(GCS_ACCESS_TOKEN_CONF)), Instant.now().plusMillis(EXPIRATION_TIME_MILLISECONDS)); } @Override diff --git a/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GcsStorageFactory.java b/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GcsStorageFactory.java index 2579685950fc..55384769f259 100644 --- a/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GcsStorageFactory.java +++ b/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GcsStorageFactory.java @@ -13,14 +13,14 @@ */ package io.trino.hdfs.gcs; -import com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.googleapis.auth.oauth2.GoogleCredential; import com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.http.HttpTransport; import com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.json.jackson2.JacksonFactory; import com.google.cloud.hadoop.repackaged.gcs.com.google.api.services.storage.Storage; +import com.google.cloud.hadoop.repackaged.gcs.com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.gcsio.GoogleCloudStorageOptions; -import com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.CredentialFactory; import com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HttpTransportFactory; import com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.RetryHttpInitializer; +import com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.RetryHttpInitializerOptions; import com.google.inject.Inject; import io.trino.hdfs.HdfsContext; import io.trino.hdfs.HdfsEnvironment; @@ -31,10 +31,10 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; -import java.time.Duration; import java.util.Optional; import static com.google.cloud.hadoop.fs.gcs.TrinoGoogleHadoopFileSystemConfiguration.getGcsOptionsBuilder; +import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.CLOUD_PLATFORM_SCOPE; import static com.google.common.base.Strings.nullToEmpty; import static io.trino.hdfs.gcs.GcsConfigurationProvider.GCS_OAUTH_KEY; import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; @@ -46,7 +46,7 @@ public class GcsStorageFactory private static final String APPLICATION_NAME = "Trino"; private final boolean useGcsAccessToken; - private final Optional jsonGoogleCredential; + private final Optional jsonGoogleCredential; @Inject public GcsStorageFactory(HiveGcsConfig hiveGcsConfig) @@ -58,12 +58,12 @@ public GcsStorageFactory(HiveGcsConfig hiveGcsConfig) String jsonKeyFilePath = hiveGcsConfig.getJsonKeyFilePath(); if (jsonKey != null) { try (InputStream inputStream = new ByteArrayInputStream(jsonKey.getBytes(UTF_8))) { - jsonGoogleCredential = Optional.of(GoogleCredential.fromStream(inputStream).createScoped(CredentialFactory.DEFAULT_SCOPES)); + jsonGoogleCredential = Optional.of(GoogleCredentials.fromStream(inputStream).createScoped(CLOUD_PLATFORM_SCOPE)); } } else if (jsonKeyFilePath != null) { try (FileInputStream inputStream = new FileInputStream(jsonKeyFilePath)) { - jsonGoogleCredential = Optional.of(GoogleCredential.fromStream(inputStream).createScoped(CredentialFactory.DEFAULT_SCOPES)); + jsonGoogleCredential = Optional.of(GoogleCredentials.fromStream(inputStream).createScoped(CLOUD_PLATFORM_SCOPE)); } } else { @@ -76,22 +76,23 @@ public Storage create(HdfsEnvironment environment, HdfsContext context, Path pat try { GoogleCloudStorageOptions gcsOptions = getGcsOptionsBuilder(environment.getConfiguration(context, path)).build(); HttpTransport httpTransport = HttpTransportFactory.createHttpTransport( - gcsOptions.getTransportType(), gcsOptions.getProxyAddress(), gcsOptions.getProxyUsername(), - gcsOptions.getProxyPassword(), - Duration.ofMillis(gcsOptions.getHttpRequestReadTimeout())); - GoogleCredential credential; + gcsOptions.getProxyPassword()); + GoogleCredentials credential; if (useGcsAccessToken) { String accessToken = nullToEmpty(context.getIdentity().getExtraCredentials().get(GCS_OAUTH_KEY)); try (ByteArrayInputStream inputStream = new ByteArrayInputStream(accessToken.getBytes(UTF_8))) { - credential = GoogleCredential.fromStream(inputStream).createScoped(CredentialFactory.DEFAULT_SCOPES); + credential = GoogleCredentials.fromStream(inputStream).createScoped(CLOUD_PLATFORM_SCOPE); } } else { credential = jsonGoogleCredential.orElseThrow(() -> new IllegalStateException("GCS credentials not configured")); } - return new Storage.Builder(httpTransport, JacksonFactory.getDefaultInstance(), new RetryHttpInitializer(credential, APPLICATION_NAME)) + return new Storage.Builder(httpTransport, JacksonFactory.getDefaultInstance(), new RetryHttpInitializer(credential, RetryHttpInitializerOptions.builder() + .setReadTimeout(gcsOptions.getHttpRequestReadTimeout()) + .setMaxRequestRetries(gcsOptions.getMaxHttpRequestRetries()) + .build())) .setApplicationName(APPLICATION_NAME) .build(); } diff --git a/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GoogleGcsConfigurationInitializer.java b/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GoogleGcsConfigurationInitializer.java index 2cb3ba0eb57e..4e979bf2e1a3 100644 --- a/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GoogleGcsConfigurationInitializer.java +++ b/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GoogleGcsConfigurationInitializer.java @@ -29,9 +29,11 @@ import java.util.Optional; import static com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystemConfiguration.GCS_CONFIG_PREFIX; -import static com.google.cloud.hadoop.fs.gcs.HadoopCredentialConfiguration.ACCESS_TOKEN_PROVIDER_IMPL_SUFFIX; -import static com.google.cloud.hadoop.fs.gcs.HadoopCredentialConfiguration.ENABLE_SERVICE_ACCOUNTS_SUFFIX; -import static com.google.cloud.hadoop.fs.gcs.HadoopCredentialConfiguration.SERVICE_ACCOUNT_JSON_KEYFILE_SUFFIX; +import static com.google.cloud.hadoop.fs.gcs.HadoopCredentialsConfiguration.SERVICE_ACCOUNT_JSON_KEYFILE_SUFFIX; +import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.ACCESS_TOKEN_PROVIDER_SUFFIX; +import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.AUTHENTICATION_TYPE_SUFFIX; +import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.AuthenticationType.ACCESS_TOKEN_PROVIDER; +import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.AuthenticationType.SERVICE_ACCOUNT_JSON_KEYFILE; import static java.nio.file.attribute.PosixFilePermission.OWNER_READ; import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE; @@ -72,12 +74,12 @@ public void initializeConfiguration(Configuration config) if (useGcsAccessToken) { // use oauth token to authenticate with Google Cloud Storage - config.setBoolean(GCS_CONFIG_PREFIX + ENABLE_SERVICE_ACCOUNTS_SUFFIX.getKey(), false); - config.setClass(GCS_CONFIG_PREFIX + ACCESS_TOKEN_PROVIDER_IMPL_SUFFIX.getKey(), GcsAccessTokenProvider.class, AccessTokenProvider.class); + config.setEnum(GCS_CONFIG_PREFIX + AUTHENTICATION_TYPE_SUFFIX.getKey(), ACCESS_TOKEN_PROVIDER); + config.setClass(GCS_CONFIG_PREFIX + ACCESS_TOKEN_PROVIDER_SUFFIX.getKey(), GcsAccessTokenProvider.class, AccessTokenProvider.class); } else if (jsonKeyFilePath != null) { // use service account key file - config.setBoolean(GCS_CONFIG_PREFIX + ENABLE_SERVICE_ACCOUNTS_SUFFIX.getKey(), true); + config.setEnum(GCS_CONFIG_PREFIX + AUTHENTICATION_TYPE_SUFFIX.getKey(), SERVICE_ACCOUNT_JSON_KEYFILE); config.set(GCS_CONFIG_PREFIX + SERVICE_ACCOUNT_JSON_KEYFILE_SUFFIX.getKey(), jsonKeyFilePath); } } diff --git a/pom.xml b/pom.xml index f340be9b53e2..7faff11c08d1 100644 --- a/pom.xml +++ b/pom.xml @@ -513,7 +513,7 @@ com.google.cloud.bigdataoss gcs-connector - hadoop3-2.2.18 + 3.0.0 shaded