diff --git a/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/DefaultGcsStorageFactory.java b/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/DefaultGcsStorageFactory.java deleted file mode 100644 index 8224dafaa6a3..000000000000 --- a/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/DefaultGcsStorageFactory.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.filesystem.gcs; - -import com.google.auth.oauth2.GoogleCredentials; -import com.google.cloud.storage.Storage; -import com.google.cloud.storage.StorageOptions; -import com.google.common.base.VerifyException; -import com.google.common.collect.ImmutableList; -import com.google.inject.Inject; -import io.trino.spi.security.ConnectorIdentity; - -import java.io.ByteArrayInputStream; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.UncheckedIOException; -import java.util.List; -import java.util.Optional; - -import static com.google.common.base.Strings.nullToEmpty; -import static java.nio.charset.StandardCharsets.UTF_8; - -public class DefaultGcsStorageFactory - implements GcsStorageFactory -{ - public static final String GCS_OAUTH_KEY = "gcs.oauth"; - public static final List DEFAULT_SCOPES = ImmutableList.of("https://www.googleapis.com/auth/cloud-platform"); - private final String projectId; - private final boolean useGcsAccessToken; - private final Optional jsonGoogleCredential; - - @Inject - public DefaultGcsStorageFactory(GcsFileSystemConfig config) - throws IOException - { - config.validate(); - projectId = config.getProjectId(); - useGcsAccessToken = config.isUseGcsAccessToken(); - String jsonKey = config.getJsonKey(); - String jsonKeyFilePath = config.getJsonKeyFilePath(); - if (jsonKey != null) { - try (InputStream inputStream = new ByteArrayInputStream(jsonKey.getBytes(UTF_8))) { - jsonGoogleCredential = Optional.of(GoogleCredentials.fromStream(inputStream).createScoped(DEFAULT_SCOPES)); - } - } - else if (jsonKeyFilePath != null) { - try (FileInputStream inputStream = new FileInputStream(jsonKeyFilePath)) { - jsonGoogleCredential = Optional.of(GoogleCredentials.fromStream(inputStream).createScoped(DEFAULT_SCOPES)); - } - } - else { - jsonGoogleCredential = Optional.empty(); - } - } - - @Override - public Storage create(ConnectorIdentity identity) - { - try { - GoogleCredentials credentials; - if (useGcsAccessToken) { - String accessToken = nullToEmpty(identity.getExtraCredentials().get(GCS_OAUTH_KEY)); - try (ByteArrayInputStream inputStream = new ByteArrayInputStream(accessToken.getBytes(UTF_8))) { - credentials = GoogleCredentials.fromStream(inputStream).createScoped(DEFAULT_SCOPES); - } - } - else { - credentials = jsonGoogleCredential.orElseThrow(() -> new VerifyException("GCS credentials not configured")); - } - StorageOptions.Builder storageOptionsBuilder = StorageOptions.newBuilder(); - if (projectId != null) { - storageOptionsBuilder.setProjectId(projectId); - } - return storageOptionsBuilder.setCredentials(credentials).build().getService(); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } -} diff --git a/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsFileSystemModule.java b/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsFileSystemModule.java index 7d4479d3557f..77c801d3281e 100644 --- a/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsFileSystemModule.java +++ b/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsFileSystemModule.java @@ -26,7 +26,7 @@ public class GcsFileSystemModule public void configure(Binder binder) { configBinder(binder).bindConfig(GcsFileSystemConfig.class); - binder.bind(GcsStorageFactory.class).to(DefaultGcsStorageFactory.class).in(Scopes.SINGLETON); + binder.bind(GcsStorageFactory.class).in(Scopes.SINGLETON); binder.bind(GcsFileSystemFactory.class).in(Scopes.SINGLETON); } } diff --git a/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsStorageFactory.java b/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsStorageFactory.java index 20db0b57b68a..176d45061fda 100644 --- a/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsStorageFactory.java +++ b/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsStorageFactory.java @@ -13,10 +13,78 @@ */ package io.trino.filesystem.gcs; +import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; +import com.google.common.base.VerifyException; +import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; import io.trino.spi.security.ConnectorIdentity; -public interface GcsStorageFactory +import java.io.ByteArrayInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.util.List; +import java.util.Optional; + +import static com.google.common.base.Strings.nullToEmpty; +import static java.nio.charset.StandardCharsets.UTF_8; + +public class GcsStorageFactory { - Storage create(ConnectorIdentity identity); + public static final String GCS_OAUTH_KEY = "gcs.oauth"; + public static final List DEFAULT_SCOPES = ImmutableList.of("https://www.googleapis.com/auth/cloud-platform"); + private final String projectId; + private final boolean useGcsAccessToken; + private final Optional jsonGoogleCredential; + + @Inject + public GcsStorageFactory(GcsFileSystemConfig config) + throws IOException + { + config.validate(); + projectId = config.getProjectId(); + useGcsAccessToken = config.isUseGcsAccessToken(); + String jsonKey = config.getJsonKey(); + String jsonKeyFilePath = config.getJsonKeyFilePath(); + if (jsonKey != null) { + try (InputStream inputStream = new ByteArrayInputStream(jsonKey.getBytes(UTF_8))) { + jsonGoogleCredential = Optional.of(GoogleCredentials.fromStream(inputStream).createScoped(DEFAULT_SCOPES)); + } + } + else if (jsonKeyFilePath != null) { + try (FileInputStream inputStream = new FileInputStream(jsonKeyFilePath)) { + jsonGoogleCredential = Optional.of(GoogleCredentials.fromStream(inputStream).createScoped(DEFAULT_SCOPES)); + } + } + else { + jsonGoogleCredential = Optional.empty(); + } + } + + public Storage create(ConnectorIdentity identity) + { + try { + GoogleCredentials credentials; + if (useGcsAccessToken) { + String accessToken = nullToEmpty(identity.getExtraCredentials().get(GCS_OAUTH_KEY)); + try (ByteArrayInputStream inputStream = new ByteArrayInputStream(accessToken.getBytes(UTF_8))) { + credentials = GoogleCredentials.fromStream(inputStream).createScoped(DEFAULT_SCOPES); + } + } + else { + credentials = jsonGoogleCredential.orElseThrow(() -> new VerifyException("GCS credentials not configured")); + } + StorageOptions.Builder storageOptionsBuilder = StorageOptions.newBuilder(); + if (projectId != null) { + storageOptionsBuilder.setProjectId(projectId); + } + return storageOptionsBuilder.setCredentials(credentials).build().getService(); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } } diff --git a/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/AbstractTestGcsFileSystem.java b/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/AbstractTestGcsFileSystem.java index 805dea0176fc..5cc129bd6029 100644 --- a/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/AbstractTestGcsFileSystem.java +++ b/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/AbstractTestGcsFileSystem.java @@ -24,11 +24,8 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.TestInstance; -import java.io.ByteArrayInputStream; import java.io.IOException; -import java.io.InputStream; import java.util.Base64; -import java.util.concurrent.TimeUnit; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Objects.requireNonNull; @@ -57,7 +54,7 @@ protected void initialize(String gcpCredentialKey) // For gcp testing this corresponds to the Cluster Storage Admin and Cluster Storage Object Admin roles byte[] jsonKeyBytes = Base64.getDecoder().decode(gcpCredentialKey); GcsFileSystemConfig config = new GcsFileSystemConfig().setJsonKey(new String(jsonKeyBytes, UTF_8)); - GcsStorageFactory storageFactory = new TestingGcsStorageFactory(config); + GcsStorageFactory storageFactory = new GcsStorageFactory(config); this.gcsFileSystemFactory = new GcsFileSystemFactory(config, storageFactory); this.storage = storageFactory.create(ConnectorIdentity.ofUser("test")); String bucket = RemoteStorageHelper.generateBucketName(); @@ -69,17 +66,20 @@ protected void initialize(String gcpCredentialKey) @AfterAll void tearDown() - throws Exception { try { - RemoteStorageHelper.forceDelete(storage, rootLocation.host().get(), 5, TimeUnit.SECONDS); - gcsFileSystemFactory.stop(); + storage.delete(rootLocation.host().get()); } finally { fileSystem = null; storage = null; rootLocation = null; - gcsFileSystemFactory = null; + try { + gcsFileSystemFactory.stop(); + } + finally { + gcsFileSystemFactory = null; + } } } @@ -126,25 +126,4 @@ protected final boolean supportsRenameFile() { return false; } - - private static class TestingGcsStorageFactory - implements GcsStorageFactory - { - private final Storage storage; - - public TestingGcsStorageFactory(GcsFileSystemConfig config) - { - requireNonNull(config, "config is null"); - InputStream inputStream = new ByteArrayInputStream(config.getJsonKey().getBytes(UTF_8)); - // Note: the default project id from the credentials file will be used. See StorageOptions.setProjectId() - RemoteStorageHelper helper = RemoteStorageHelper.create(null, inputStream); - this.storage = helper.getOptions().getService(); - } - - @Override - public Storage create(ConnectorIdentity identity) - { - return storage; - } - } } diff --git a/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/TestGcsFileSystemGcs.java b/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/TestGcsFileSystemGcs.java index 32c7cfb8ff15..b9a056e487fc 100644 --- a/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/TestGcsFileSystemGcs.java +++ b/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/TestGcsFileSystemGcs.java @@ -14,12 +14,10 @@ package io.trino.filesystem.gcs; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.TestInstance; import java.io.IOException; -@Disabled // TODO Re-enable once fixed the initialization failure https://github.com/trinodb/trino/issues/19785 @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class TestGcsFileSystemGcs extends AbstractTestGcsFileSystem