diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java index 99e736f798..2788567da7 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java @@ -378,7 +378,7 @@ public void testIcebergCreateTablesInExternalCatalog() throws IOException { .withPartitionSpec(PartitionSpec.unpartitioned()) .create()) .isInstanceOf(BadRequestException.class) - .hasMessage("Malformed request: Cannot create table on external catalogs."); + .hasMessage("Malformed request: Cannot create table on static-facade external catalogs."); } } @@ -515,7 +515,7 @@ public void testIcebergUpdateTableInExternalCatalog() throws IOException { 10L)) .commit()) .isInstanceOf(BadRequestException.class) - .hasMessage("Malformed request: Cannot update table on external catalogs."); + .hasMessage("Malformed request: Cannot update table on static-facade external catalogs."); } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java index d03a55b2f3..f857d03acd 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Optional; import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.persistence.cache.EntityWeigher; /** @@ -36,6 +37,22 @@ protected FeatureConfiguration( super(key, description, defaultValue, catalogConfig); } + /** + * Helper for the common scenario of gating a feature with a boolean FeatureConfiguration, where + * we want to throw an UnsupportedOperationException if it's not enabled. + */ + public static void enforceFeatureEnabledOrThrow( + CallContext callContext, FeatureConfiguration featureConfig) { + boolean enabled = + callContext + .getPolarisCallContext() + .getConfigurationStore() + .getConfiguration(callContext.getPolarisCallContext(), featureConfig); + if (!enabled) { + throw new UnsupportedOperationException("Feature not enabled: " + featureConfig.key); + } + } + public static final FeatureConfiguration ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING = PolarisConfiguration.builder() @@ -201,4 +218,13 @@ protected FeatureConfiguration( + " requires experimentation in the specific deployment environment") .defaultValue(100 * EntityWeigher.WEIGHT_PER_MB) .buildFeatureConfiguration(); + + public static final FeatureConfiguration ENABLE_CATALOG_FEDERATION = + PolarisConfiguration.builder() + .key("ENABLE_CATALOG_FEDERATION") + .description( + "If true, allows creating and using ExternalCatalogs containing ConnectionConfigInfos" + + " to perform federation to remote catalogs.") + .defaultValue(false) + .buildFeatureConfiguration(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationParametersDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationParametersDpo.java new file mode 100644 index 0000000000..35687feeb9 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationParametersDpo.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.connection; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import java.util.Map; +import org.apache.polaris.core.admin.model.AuthenticationParameters; +import org.apache.polaris.core.admin.model.BearerAuthenticationParameters; +import org.apache.polaris.core.admin.model.OAuthClientCredentialsParameters; +import org.apache.polaris.core.secrets.UserSecretReference; + +/** + * The internal persistence-object counterpart to AuthenticationParameters defined in the API model. + * Important: JsonSubTypes must be kept in sync with {@link AuthenticationType}. + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "authenticationTypeCode", visible = true) +@JsonSubTypes({ + @JsonSubTypes.Type(value = OAuthClientCredentialsParametersDpo.class, name = "1"), + @JsonSubTypes.Type(value = BearerAuthenticationParametersDpo.class, name = "2"), +}) +public abstract class AuthenticationParametersDpo implements IcebergCatalogPropertiesProvider { + + public static final String INLINE_CLIENT_SECRET_REFERENCE_KEY = "inlineClientSecretReference"; + public static final String INLINE_BEARER_TOKEN_REFERENCE_KEY = "inlineBearerTokenReference"; + + @JsonProperty(value = "authenticationTypeCode") + private final int authenticationTypeCode; + + public AuthenticationParametersDpo( + @JsonProperty(value = "authenticationTypeCode", required = true) int authenticationTypeCode) { + this.authenticationTypeCode = authenticationTypeCode; + } + + public int getAuthenticationTypeCode() { + return authenticationTypeCode; + } + + public abstract AuthenticationParameters asAuthenticationParametersModel(); + + public static AuthenticationParametersDpo fromAuthenticationParametersModelWithSecrets( + AuthenticationParameters authenticationParameters, + Map secretReferences) { + final AuthenticationParametersDpo config; + switch (authenticationParameters.getAuthenticationType()) { + case OAUTH: + OAuthClientCredentialsParameters oauthClientCredentialsModel = + (OAuthClientCredentialsParameters) authenticationParameters; + config = + new OAuthClientCredentialsParametersDpo( + AuthenticationType.OAUTH.getCode(), + oauthClientCredentialsModel.getTokenUri(), + oauthClientCredentialsModel.getClientId(), + secretReferences.get(INLINE_CLIENT_SECRET_REFERENCE_KEY), + oauthClientCredentialsModel.getScopes()); + break; + case BEARER: + BearerAuthenticationParameters bearerAuthenticationParametersModel = + (BearerAuthenticationParameters) authenticationParameters; + config = + new BearerAuthenticationParametersDpo( + AuthenticationType.BEARER.getCode(), + secretReferences.get(INLINE_BEARER_TOKEN_REFERENCE_KEY)); + break; + default: + throw new IllegalStateException( + "Unsupported authentication type: " + authenticationParameters.getAuthenticationType()); + } + return config; + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationType.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationType.java new file mode 100644 index 0000000000..8ccc529854 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationType.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.connection; + +/** + * The internal persistence-object counterpart to AuthenticationParameters.AuthenticationTypeEnum + * defined in the API model. We define integer type codes in this enum for better compatibility + * within persisted data in case the names of enum types are ever changed in place. + * + *

Important: Codes must be kept in-sync with JsonSubTypes annotated within {@link + * AuthenticationParametersDpo}. + */ +public enum AuthenticationType { + OAUTH(1), + BEARER(2); + + private final int code; + + AuthenticationType(int code) { + this.code = code; + } + + public int getCode() { + return this.code; + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/BearerAuthenticationParametersDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/BearerAuthenticationParametersDpo.java new file mode 100644 index 0000000000..b5d136bc5d --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/BearerAuthenticationParametersDpo.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.connection; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; +import jakarta.annotation.Nonnull; +import java.util.Map; +import org.apache.iceberg.rest.auth.OAuth2Properties; +import org.apache.polaris.core.admin.model.AuthenticationParameters; +import org.apache.polaris.core.admin.model.BearerAuthenticationParameters; +import org.apache.polaris.core.secrets.UserSecretReference; +import org.apache.polaris.core.secrets.UserSecretsManager; + +/** + * The internal persistence-object counterpart to BearerAuthenticationParameters defined in the API + * model. + */ +public class BearerAuthenticationParametersDpo extends AuthenticationParametersDpo { + + @JsonProperty(value = "bearerTokenReference") + private final UserSecretReference bearerTokenReference; + + public BearerAuthenticationParametersDpo( + @JsonProperty(value = "authenticationTypeCode", required = true) int authenticationTypeCode, + @JsonProperty(value = "bearerTokenReference", required = true) @Nonnull + UserSecretReference bearerTokenReference) { + super(authenticationTypeCode); + this.bearerTokenReference = bearerTokenReference; + } + + public @Nonnull UserSecretReference getBearerTokenReference() { + return bearerTokenReference; + } + + @Override + public @Nonnull Map asIcebergCatalogProperties( + UserSecretsManager secretsManager) { + String bearerToken = secretsManager.readSecret(getBearerTokenReference()); + return Map.of(OAuth2Properties.TOKEN, bearerToken); + } + + @Override + public AuthenticationParameters asAuthenticationParametersModel() { + return BearerAuthenticationParameters.builder() + .setAuthenticationType(AuthenticationParameters.AuthenticationTypeEnum.BEARER) + .build(); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("authenticationTypeCode", getAuthenticationTypeCode()) + .add("bearerTokenReference", getBearerTokenReference()) + .toString(); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java new file mode 100644 index 0000000000..c5bf56f292 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.connection; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.util.Map; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.admin.model.ConnectionConfigInfo; +import org.apache.polaris.core.admin.model.IcebergRestConnectionConfigInfo; +import org.apache.polaris.core.secrets.UserSecretReference; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The internal persistence-object counterpart to ConnectionConfigInfo defined in the API model. + * Important: JsonSubTypes must be kept in sync with {@link ConnectionType}. + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "connectionTypeCode", visible = true) +@JsonSubTypes({ + @JsonSubTypes.Type(value = IcebergRestConnectionConfigInfoDpo.class, name = "1"), +}) +public abstract class ConnectionConfigInfoDpo implements IcebergCatalogPropertiesProvider { + private static final Logger logger = LoggerFactory.getLogger(ConnectionConfigInfoDpo.class); + + // The type of the connection + private final int connectionTypeCode; + + // The URI of the remote catalog + private final String uri; + + // The authentication parameters for the connection + private final AuthenticationParametersDpo authenticationParameters; + + public ConnectionConfigInfoDpo( + @JsonProperty(value = "connectionTypeCode", required = true) int connectionTypeCode, + @JsonProperty(value = "uri", required = true) @Nonnull String uri, + @JsonProperty(value = "authenticationParameters", required = true) @Nonnull + AuthenticationParametersDpo authenticationParameters) { + this(connectionTypeCode, uri, authenticationParameters, true); + } + + protected ConnectionConfigInfoDpo( + int connectionTypeCode, + @Nonnull String uri, + @Nonnull AuthenticationParametersDpo authenticationParameters, + boolean validateUri) { + this.connectionTypeCode = connectionTypeCode; + this.uri = uri; + this.authenticationParameters = authenticationParameters; + if (validateUri) { + validateUri(uri); + } + } + + public int getConnectionTypeCode() { + return connectionTypeCode; + } + + public String getUri() { + return uri; + } + + public AuthenticationParametersDpo getAuthenticationParameters() { + return authenticationParameters; + } + + private static final ObjectMapper DEFAULT_MAPPER; + + static { + DEFAULT_MAPPER = new ObjectMapper(); + DEFAULT_MAPPER.setSerializationInclusion(JsonInclude.Include.NON_NULL); + DEFAULT_MAPPER.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + } + + public String serialize() { + try { + return DEFAULT_MAPPER.writeValueAsString(this); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public static @Nullable ConnectionConfigInfoDpo deserialize( + @Nonnull PolarisDiagnostics diagnostics, final @Nonnull String jsonStr) { + try { + return DEFAULT_MAPPER.readValue(jsonStr, ConnectionConfigInfoDpo.class); + } catch (JsonProcessingException exception) { + diagnostics.fail( + "fail_to_deserialize_connection_configuration", exception, "jsonStr={}", jsonStr); + } + return null; + } + + /** Validates the remote URI. */ + protected void validateUri(String uri) { + try { + URI uriObj = URI.create(uri); + URL url = uriObj.toURL(); + } catch (IllegalArgumentException | MalformedURLException e) { + throw new IllegalArgumentException("Invalid remote URI: " + uri, e); + } + } + + /** + * Converts from the API-model ConnectionConfigInfo by merging basic carryover fields with + * expected associated secretReference(s) that have been previously scrubbed or resolved from + * inline secret fields of the API request. + */ + public static ConnectionConfigInfoDpo fromConnectionConfigInfoModelWithSecrets( + ConnectionConfigInfo connectionConfigurationModel, + Map secretReferences) { + ConnectionConfigInfoDpo config = null; + switch (connectionConfigurationModel.getConnectionType()) { + case ICEBERG_REST: + IcebergRestConnectionConfigInfo icebergRestConfigModel = + (IcebergRestConnectionConfigInfo) connectionConfigurationModel; + AuthenticationParametersDpo authenticationParameters = + AuthenticationParametersDpo.fromAuthenticationParametersModelWithSecrets( + icebergRestConfigModel.getAuthenticationParameters(), secretReferences); + config = + new IcebergRestConnectionConfigInfoDpo( + ConnectionType.ICEBERG_REST.getCode(), + icebergRestConfigModel.getUri(), + authenticationParameters, + icebergRestConfigModel.getRemoteCatalogName()); + break; + default: + throw new IllegalStateException( + "Unsupported connection type: " + connectionConfigurationModel.getConnectionType()); + } + return config; + } + + /** + * Produces the correponding API-model ConnectionConfigInfo for this persistence object; many + * fields are one-to-one direct mappings, but some fields, such as secretReferences, might only be + * applicable/present in the persistence object, but not the API model object. + */ + public abstract ConnectionConfigInfo asConnectionConfigInfoModel(); +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionType.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionType.java new file mode 100644 index 0000000000..ef49cb5cab --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionType.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.connection; + +import jakarta.annotation.Nullable; + +/** + * The internal persistence-object counterpart to ConnectionConfigInfo.ConnectionTypeEnum defined in + * the API model. We define integer type codes in this enum for better compatibility within + * persisted data in case the names of enum types are ever changed in place. + * + *

Important: Codes must be kept in-sync with JsonSubTypes annotated within {@link + * ConnectionConfigInfoDpo}. + */ +public enum ConnectionType { + ICEBERG_REST(1); + + private static final ConnectionType[] REVERSE_MAPPING_ARRAY; + + static { + // find max array size + int maxId = 0; + for (ConnectionType connectionType : ConnectionType.values()) { + if (maxId < connectionType.code) { + maxId = connectionType.code; + } + } + + // allocate mapping array + REVERSE_MAPPING_ARRAY = new ConnectionType[maxId + 1]; + + // populate mapping array + for (ConnectionType connectionType : ConnectionType.values()) { + REVERSE_MAPPING_ARRAY[connectionType.code] = connectionType; + } + } + + private final int code; + + ConnectionType(int code) { + this.code = code; + } + + /** + * Given the code associated to the type, return the associated ConnectionType. Return null if not + * found + * + * @param connectionTypeCode code associated to the entity type + * @return ConnectionType corresponding to that code or null if mapping not found + */ + public static @Nullable ConnectionType fromCode(int connectionTypeCode) { + // ensure it is within bounds + if (connectionTypeCode >= REVERSE_MAPPING_ARRAY.length) { + return null; + } + + // get value + return REVERSE_MAPPING_ARRAY[connectionTypeCode]; + } + + public int getCode() { + return this.code; + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergCatalogPropertiesProvider.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergCatalogPropertiesProvider.java new file mode 100644 index 0000000000..e7955bc61a --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergCatalogPropertiesProvider.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.connection; + +import jakarta.annotation.Nonnull; +import java.util.Map; +import org.apache.polaris.core.secrets.UserSecretsManager; + +/** + * Configuration wrappers which ultimately translate their contents into Iceberg properties and + * which may hold other nested configuration wrapper objects implement this interface to allow + * delegating type-specific configuration translation logic to subclasses instead of needing to + * expose the internals of deeply nested configuration objects to a visitor class. + */ +public interface IcebergCatalogPropertiesProvider { + @Nonnull + Map asIcebergCatalogProperties(UserSecretsManager secretsManager); +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigInfoDpo.java new file mode 100644 index 0000000000..2ebe099519 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigInfoDpo.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.connection; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.HashMap; +import java.util.Map; +import org.apache.iceberg.CatalogProperties; +import org.apache.polaris.core.admin.model.ConnectionConfigInfo; +import org.apache.polaris.core.admin.model.IcebergRestConnectionConfigInfo; +import org.apache.polaris.core.secrets.UserSecretsManager; + +/** + * The internal persistence-object counterpart to IcebergRestConnectionConfigInfo defined in the API + * model. + */ +public class IcebergRestConnectionConfigInfoDpo extends ConnectionConfigInfoDpo + implements IcebergCatalogPropertiesProvider { + + private final String remoteCatalogName; + + public IcebergRestConnectionConfigInfoDpo( + @JsonProperty(value = "connectionTypeCode", required = true) int connectionTypeCode, + @JsonProperty(value = "uri", required = true) @Nonnull String uri, + @JsonProperty(value = "authenticationParameters", required = true) @Nonnull + AuthenticationParametersDpo authenticationParameters, + @JsonProperty(value = "remoteCatalogName", required = false) @Nullable + String remoteCatalogName) { + super(connectionTypeCode, uri, authenticationParameters); + this.remoteCatalogName = remoteCatalogName; + } + + public String getRemoteCatalogName() { + return remoteCatalogName; + } + + @Override + public @Nonnull Map asIcebergCatalogProperties( + UserSecretsManager secretsManager) { + HashMap properties = new HashMap<>(); + properties.put(CatalogProperties.URI, getUri()); + if (getRemoteCatalogName() != null) { + properties.put(CatalogProperties.WAREHOUSE_LOCATION, getRemoteCatalogName()); + } + properties.putAll(getAuthenticationParameters().asIcebergCatalogProperties(secretsManager)); + return properties; + } + + @Override + public ConnectionConfigInfo asConnectionConfigInfoModel() { + return IcebergRestConnectionConfigInfo.builder() + .setConnectionType(ConnectionConfigInfo.ConnectionTypeEnum.ICEBERG_REST) + .setUri(getUri()) + .setRemoteCatalogName(getRemoteCatalogName()) + .setAuthenticationParameters( + getAuthenticationParameters().asAuthenticationParametersModel()) + .build(); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("connectionTypeCode", getConnectionTypeCode()) + .add("uri", getUri()) + .add("remoteCatalogName", getRemoteCatalogName()) + .add("authenticationParameters", getAuthenticationParameters().toString()) + .toString(); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/OAuthClientCredentialsParametersDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/OAuthClientCredentialsParametersDpo.java new file mode 100644 index 0000000000..127ed3e3c4 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/OAuthClientCredentialsParametersDpo.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.connection; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Joiner; +import com.google.common.base.MoreObjects; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import org.apache.iceberg.rest.auth.OAuth2Properties; +import org.apache.iceberg.rest.auth.OAuth2Util; +import org.apache.polaris.core.admin.model.AuthenticationParameters; +import org.apache.polaris.core.admin.model.OAuthClientCredentialsParameters; +import org.apache.polaris.core.secrets.UserSecretReference; +import org.apache.polaris.core.secrets.UserSecretsManager; + +/** + * The internal persistence-object counterpart to OAuthClientCredentialsParameters defined in the + * API model. + */ +public class OAuthClientCredentialsParametersDpo extends AuthenticationParametersDpo { + + private static final Joiner COLON_JOINER = Joiner.on(":"); + + @JsonProperty(value = "tokenUri") + private final String tokenUri; + + @JsonProperty(value = "clientId") + private final String clientId; + + @JsonProperty(value = "clientSecretReference") + private final UserSecretReference clientSecretReference; + + @JsonProperty(value = "scopes") + private final List scopes; + + public OAuthClientCredentialsParametersDpo( + @JsonProperty(value = "authenticationTypeCode", required = true) int authenticationTypeCode, + @JsonProperty(value = "tokenUri", required = false) @Nullable String tokenUri, + @JsonProperty(value = "clientId", required = true) @Nonnull String clientId, + @JsonProperty(value = "clientSecretReference", required = true) @Nonnull + UserSecretReference clientSecretReference, + @JsonProperty(value = "scopes", required = false) @Nullable List scopes) { + super(authenticationTypeCode); + + this.tokenUri = tokenUri; + this.clientId = clientId; + this.clientSecretReference = clientSecretReference; + this.scopes = scopes; + + validateTokenUri(tokenUri); + } + + public @Nullable String getTokenUri() { + return tokenUri; + } + + public @Nonnull String getClientId() { + return clientId; + } + + public @Nonnull UserSecretReference getClientSecretReference() { + return clientSecretReference; + } + + public @Nonnull List getScopes() { + return scopes; + } + + @JsonIgnore + public @Nonnull String getScopesAsString() { + return OAuth2Util.toScope( + Objects.requireNonNullElse(scopes, List.of(OAuth2Properties.CATALOG_SCOPE))); + } + + @JsonIgnore + private @Nonnull String getCredentialAsConcatenatedString(UserSecretsManager secretsManager) { + String clientSecret = secretsManager.readSecret(getClientSecretReference()); + return COLON_JOINER.join(clientId, clientSecret); + } + + @Override + public @Nonnull Map asIcebergCatalogProperties( + UserSecretsManager secretsManager) { + HashMap properties = new HashMap<>(); + if (getTokenUri() != null) { + properties.put(OAuth2Properties.OAUTH2_SERVER_URI, getTokenUri()); + } + properties.put(OAuth2Properties.CREDENTIAL, getCredentialAsConcatenatedString(secretsManager)); + properties.put(OAuth2Properties.SCOPE, getScopesAsString()); + return properties; + } + + @Override + public AuthenticationParameters asAuthenticationParametersModel() { + return OAuthClientCredentialsParameters.builder() + .setAuthenticationType(AuthenticationParameters.AuthenticationTypeEnum.OAUTH) + .setTokenUri(getTokenUri()) + .setClientId(getClientId()) + .setScopes(getScopes()) + .build(); + } + + /** Validates the token URI. */ + protected void validateTokenUri(String tokenUri) { + if (tokenUri == null) { + return; + } + + try { + URI uri = URI.create(tokenUri); + URL url = uri.toURL(); + } catch (IllegalArgumentException | MalformedURLException e) { + throw new IllegalArgumentException("Invalid token URI: " + tokenUri, e); + } + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("authenticationTypeCode", getAuthenticationTypeCode()) + .add("tokenUri", getTokenUri()) + .add("clientId", getClientId()) + .add("clientSecretReference", getClientSecretReference()) + .add("scopes", getScopesAsString()) + .toString(); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java index a49bba9e6e..a995883304 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java @@ -35,12 +35,15 @@ import org.apache.polaris.core.admin.model.AzureStorageConfigInfo; import org.apache.polaris.core.admin.model.Catalog; import org.apache.polaris.core.admin.model.CatalogProperties; +import org.apache.polaris.core.admin.model.ConnectionConfigInfo; import org.apache.polaris.core.admin.model.ExternalCatalog; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.GcpStorageConfigInfo; import org.apache.polaris.core.admin.model.PolarisCatalog; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.config.BehaviorChangeConfiguration; +import org.apache.polaris.core.connection.ConnectionConfigInfoDpo; +import org.apache.polaris.core.secrets.UserSecretReference; import org.apache.polaris.core.storage.FileStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; @@ -78,7 +81,6 @@ public static CatalogEntity of(PolarisBaseEntity sourceEntity) { } public static CatalogEntity fromCatalog(Catalog catalog) { - Builder builder = new Builder() .setName(catalog.getName()) @@ -121,6 +123,7 @@ public Catalog asCatalog() { .setLastUpdateTimestamp(getLastUpdateTimestamp()) .setEntityVersion(getEntityVersion()) .setStorageConfigInfo(getStorageInfo(internalProperties)) + .setConnectionConfigInfo(getConnectionInfo(internalProperties)) .build(); } @@ -166,6 +169,15 @@ private StorageConfigInfo getStorageInfo(Map internalProperties) return null; } + private ConnectionConfigInfo getConnectionInfo(Map internalProperties) { + if (internalProperties.containsKey( + PolarisEntityConstants.getConnectionConfigInfoPropertyName())) { + ConnectionConfigInfoDpo configInfo = getConnectionConfigInfoDpo(); + return configInfo.asConnectionConfigInfoModel(); + } + return null; + } + public String getDefaultBaseLocation() { return getPropertiesAsMap().get(DEFAULT_BASE_LOCATION_KEY); } @@ -190,6 +202,21 @@ public Catalog.TypeEnum getCatalogType() { .orElse(null); } + public boolean isPassthroughFacade() { + return getInternalPropertiesAsMap() + .containsKey(PolarisEntityConstants.getConnectionConfigInfoPropertyName()); + } + + public ConnectionConfigInfoDpo getConnectionConfigInfoDpo() { + String configStr = + getInternalPropertiesAsMap() + .get(PolarisEntityConstants.getConnectionConfigInfoPropertyName()); + if (configStr != null) { + return ConnectionConfigInfoDpo.deserialize(new PolarisDefaultDiagServiceImpl(), configStr); + } + return null; + } + public static class Builder extends PolarisEntity.BaseBuilder { public Builder() { super(); @@ -290,6 +317,19 @@ private void validateMaxAllowedLocations(Collection allowedLocations) { } } + public Builder setConnectionConfigInfoDpoWithSecrets( + ConnectionConfigInfo connectionConfigurationModel, + Map secretReferences) { + if (connectionConfigurationModel != null) { + ConnectionConfigInfoDpo config = + ConnectionConfigInfoDpo.fromConnectionConfigInfoModelWithSecrets( + connectionConfigurationModel, secretReferences); + internalProperties.put( + PolarisEntityConstants.getConnectionConfigInfoPropertyName(), config.serialize()); + } + return this; + } + @Override public CatalogEntity build() { return new CatalogEntity(buildBase()); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntity.java index 08ac29b350..363c1d3ce4 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntity.java @@ -409,6 +409,11 @@ public B setInternalProperties(Map internalProperties) { return (B) this; } + public B addInternalProperty(String key, String value) { + this.internalProperties.put(key, value); + return (B) this; + } + public B setEntityVersion(int entityVersion) { this.entityVersion = entityVersion; return (B) this; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityConstants.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityConstants.java index 26d3c09a73..4452a90200 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityConstants.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityConstants.java @@ -52,6 +52,9 @@ public class PolarisEntityConstants { private static final String STORAGE_INTEGRATION_IDENTIFIER_PROPERTY_NAME = "storage_integration_identifier"; + private static final String CONNECTION_CONFIGURATION_INFO_PROPERTY_NAME = + "connection_configuration_info"; + private static final String PRINCIPAL_TYPE_NAME = "principal_type_name"; public static final String PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_STATE = @@ -104,6 +107,10 @@ public static String getStorageConfigInfoPropertyName() { return STORAGE_CONFIGURATION_INFO_PROPERTY_NAME; } + public static String getConnectionConfigInfoPropertyName() { + return CONNECTION_CONFIGURATION_INFO_PROPERTY_NAME; + } + public static String getPolarisStorageIntegrationNameFormat() { return POLARIS_STORAGE_INT_NAME_FORMAT; } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/PolarisResolutionManifest.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/PolarisResolutionManifest.java index fd000a167a..4b2d5ce96e 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/PolarisResolutionManifest.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/PolarisResolutionManifest.java @@ -160,6 +160,10 @@ public ResolverStatus resolveAll() { return primaryResolverStatus; } + public boolean getIsPassthroughFacade() { + return primaryResolver.getIsPassthroughFacade(); + } + @Override public PolarisResolvedPathWrapper getResolvedReferenceCatalogEntity() { return getResolvedReferenceCatalogEntity(false); @@ -212,7 +216,12 @@ public PolarisResolvedPathWrapper getPassthroughResolvedPath(Object key) { } List resolvedPath = passthroughResolver.getResolvedPath(); - if (requestedPath.isOptional()) { + // If the catalog is a passthrough facade, we can go ahead and just return only as much of + // the parent path as was successfully found. + // TODO: For passthrough facade semantics, consider whether this should be where we generate + // the JIT-created entities that would get committed after we find them in the remote + // catalog. + if (requestedPath.isOptional() && !getIsPassthroughFacade()) { if (resolvedPath.size() != requestedPath.getEntityNames().size()) { LOGGER.debug( "Returning null for key {} due to size mismatch from getPassthroughResolvedPath " @@ -355,7 +364,12 @@ public PolarisResolvedPathWrapper getResolvedPath(Object key, boolean prependRoo // Return null for a partially-resolved "optional" path. ResolverPath requestedPath = addedPaths.get(index); List resolvedPath = primaryResolver.getResolvedPaths().get(index); - if (requestedPath.isOptional()) { + // If the catalog is a passthrough facade, we can go ahead and just return only as much of + // the parent path as was successfully found. + // TODO: For passthrough facade semantics, consider whether this should be where we generate + // the JIT-created entities that would get committed after we find them in the remote + // catalog. + if (requestedPath.isOptional() && !getIsPassthroughFacade()) { if (resolvedPath.size() != requestedPath.getEntityNames().size()) { return null; } @@ -383,7 +397,13 @@ public PolarisResolvedPathWrapper getResolvedPath( if (resolvedPath == null) { return null; } - if (resolvedPath.getRawLeafEntity() != null + // In the case of a passthrough facade, we may have only resolved part of the parent path + // in which case the subtype wouldn't match; return the path anyways in this case. + // + // TODO: Reconcile how we'll handle "TABLE_NOT_FOUND" or "VIEW_NOT_FOUND" semantics + // against the remote catalog. + if (!getIsPassthroughFacade() + && resolvedPath.getRawLeafEntity() != null && subType != PolarisEntitySubType.ANY_SUBTYPE && resolvedPath.getRawLeafEntity().getSubType() != subType) { return null; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/Resolver.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/Resolver.java index b1c7fc1697..85d3e74c83 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/Resolver.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/Resolver.java @@ -33,6 +33,7 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; +import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntityConstants; @@ -106,6 +107,10 @@ public class Resolver { private ResolverStatus resolverStatus; + // Set if we determine the reference catalog is a passthrough facade, which impacts + // leniency of resolution of in-catalog paths + private boolean isPassthroughFacade; + /** * Constructor, effectively starts an entity resolver session * @@ -264,6 +269,10 @@ public ResolverStatus resolveAll() { return status; } + public boolean getIsPassthroughFacade() { + return this.isPassthroughFacade; + } + /** * @return the principal we resolved */ @@ -656,6 +665,7 @@ private ResolverStatus resolveEntities( this.resolveByName(toValidate, entityName.getEntityType(), entityName.getEntityName()); // if not found, we can exit unless the entity is optional + // TODO: Consider how this interacts with CATALOG_ROLE in the isPassthroughFacade case. if (!entityName.isOptional() && (resolvedEntity == null || resolvedEntity.getEntity().isDropped())) { return new ResolverStatus(entityName.getEntityType(), entityName.getEntityName()); @@ -706,7 +716,9 @@ private ResolverStatus resolvePaths( // if not found, abort if (segment == null || segment.getEntity().isDropped()) { - if (path.isOptional()) { + // If we've determined the catalog is a passthrough facade, treat all paths as + // optional. + if (path.isOptional() || this.isPassthroughFacade) { // we have resolved as much as what we could have break; } else { @@ -851,6 +863,10 @@ private ResolverStatus resolveReferenceCatalog( } } + if (CatalogEntity.of(this.resolvedReferenceCatalog.getEntity()).isPassthroughFacade()) { + this.isPassthroughFacade = true; + } + // all good return new ResolverStatus(ResolverStatus.StatusEnum.SUCCESS); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManager.java b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManager.java new file mode 100644 index 0000000000..22e8c53786 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManager.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.secrets; + +import jakarta.annotation.Nonnull; +import java.io.UnsupportedEncodingException; +import java.security.SecureRandom; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.polaris.core.entity.PolarisEntityCore; + +/** + * A minimal in-memory implementation of UserSecretsManager that should only be used for test and + * development purposes. + */ +public class UnsafeInMemorySecretsManager implements UserSecretsManager { + // TODO: Remove this and wire into QuarkusProducers; just a placeholder for now to get the + // rest of the logic working. + public static final UserSecretsManager GLOBAL_INSTANCE = new UnsafeInMemorySecretsManager(); + + private final Map rawSecretStore = new ConcurrentHashMap<>(); + private final SecureRandom rand = new SecureRandom(); + + // Keys for information stored in referencePayload + private static final String CIPHERTEXT_HASH = "ciphertext-hash"; + private static final String ENCRYPTION_KEY = "encryption-key"; + + /** {@inheritDoc} */ + @Override + @Nonnull + public UserSecretReference writeSecret( + @Nonnull String secret, @Nonnull PolarisEntityCore forEntity) { + // For illustrative purposes and to exercise the control flow of requiring both the stored + // secret as well as the secretReferencePayload to recover the original secret, we'll use + // basic XOR encryption and store the randomly generated key in the reference payload. + // A production implementation will typically use a standard crypto library if applicable. + byte[] secretBytes; + try { + secretBytes = secret.getBytes("UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + byte[] oneTimeKey = new byte[secretBytes.length]; + byte[] cipherTextBytes = new byte[secretBytes.length]; + + // Generate one-time key of length equal to the original secret's bytes. + rand.nextBytes(oneTimeKey); + + // XOR all the bytes to generate the cipherText + for (int i = 0; i < cipherTextBytes.length; ++i) { + cipherTextBytes[i] = (byte) (secretBytes[i] ^ oneTimeKey[i]); + } + + // Store as Base64 since raw bytes won't play well with non-invertible String behaviors + // related to charset encodings. + String encryptedSecretCipherTextBase64 = Base64.getEncoder().encodeToString(cipherTextBytes); + String encryptedSecretKeyBase64 = Base64.getEncoder().encodeToString(oneTimeKey); + + String secretUrn; + for (int secretOrdinal = 0; ; ++secretOrdinal) { + secretUrn = + String.format( + "urn:polaris-secret:unsafe-in-memory:%d:%d", forEntity.getId(), secretOrdinal); + + // Store the base64-encoded encrypted ciphertext in the simulated "secret store". + String existingSecret = + rawSecretStore.putIfAbsent(secretUrn, encryptedSecretCipherTextBase64); + + // If there was already something stored under the current URN, continue to loop with + // an incremented ordinal suffix until we find an unused URN. + if (existingSecret == null) { + break; + } + } + + Map referencePayload = new HashMap<>(); + + // Keep a hash to detect data corruption or tampering; hash the base64-encoded string so + // we detect the corruption even before attempting to base64-decode it. + referencePayload.put(CIPHERTEXT_HASH, DigestUtils.sha256Hex(encryptedSecretCipherTextBase64)); + + // Keep the randomly generated one-time-use encryption key in the reference payload. + // A production implementation may choose to store an encryption key reference or URN if the + // key is ever shared and/or the key isn't a one-time-pad of the same length as the source + // secret. + referencePayload.put(ENCRYPTION_KEY, encryptedSecretKeyBase64); + UserSecretReference secretReference = new UserSecretReference(secretUrn, referencePayload); + return secretReference; + } + + /** {@inheritDoc} */ + @Override + @Nonnull + public String readSecret(@Nonnull UserSecretReference secretReference) { + // TODO: Precondition checks and/or wire in PolarisDiagnostics + String encryptedSecretCipherTextBase64 = rawSecretStore.get(secretReference.getUrn()); + if (encryptedSecretCipherTextBase64 == null) { + // Secret at this URN no longer exists. + return null; + } + + String encryptedSecretKeyBase64 = secretReference.getReferencePayload().get(ENCRYPTION_KEY); + + // Validate integrity of the base64-encoded ciphertext which was retrieved from the secret + // store against the hash we stored in the referencePayload. + String expecteCipherTextBase64Hash = secretReference.getReferencePayload().get(CIPHERTEXT_HASH); + String retrievedCipherTextBase64Hash = DigestUtils.sha256Hex(encryptedSecretCipherTextBase64); + if (!Objects.equals(retrievedCipherTextBase64Hash, expecteCipherTextBase64Hash)) { + throw new IllegalArgumentException( + String.format( + "Ciphertext hash mismatch for URN %s; expected %s got %s", + secretReference.getUrn(), + expecteCipherTextBase64Hash, + retrievedCipherTextBase64Hash)); + } + + byte[] cipherTextBytes = Base64.getDecoder().decode(encryptedSecretCipherTextBase64); + byte[] oneTimeKey = Base64.getDecoder().decode(encryptedSecretKeyBase64); + byte[] secretBytes = new byte[cipherTextBytes.length]; + + // XOR all the bytes to recover the secret + for (int i = 0; i < cipherTextBytes.length; ++i) { + secretBytes[i] = (byte) (cipherTextBytes[i] ^ oneTimeKey[i]); + } + + try { + return new String(secretBytes, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + + /** {@inheritDoc} */ + @Override + public void deleteSecret(@Nonnull UserSecretReference secretReference) { + rawSecretStore.remove(secretReference.getUrn()); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretReference.java b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretReference.java new file mode 100644 index 0000000000..7181acb041 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretReference.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.secrets; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; +import com.google.common.base.Preconditions; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * Represents a "wrapped reference" to a user-owned secret that holds an identifier to retrieve + * possibly remotely-stored secret material, along with an open-ended "referencePayload" that is + * specific to an implementation of the secret storage and which is needed "unwrap" the actual + * secret in combination with whatever is stored in the remote secrets storage. + * + *

Example scenarios: + * + *

If an implementation simply stores secrets directly in the secrets manager, the + * referencePayload may be empty and "unwrapping" would be a simple identity/no-op transformation. + * + *

If tampering or corruption of secrets in the secrets manager presents a unique threat, an + * implementation may use the referencePayload to ensure data integrity of the secret by storing a + * checksum or hash of the stored secret. + * + *

If the system must protect against independent exfiltration/attacks on a dedicated secrets + * manager and the core persistence database, the referencePayload may be used to coordinate + * secondary encryption keys such that the original secret can only be fully "unwrapped" given both + * the stored "secret material" as well as the referencePayload and any associated keys used for + * encryption. + */ +public class UserSecretReference { + @JsonProperty(value = "urn") + private final String urn; + + @JsonProperty(value = "referencePayload") + private final Map referencePayload; + + /** + * @param urn A string which should be self-sufficient to retrieve whatever secret material that + * is stored in the remote secret store and also to identify an implementation of the + * UserSecretsManager which is capable of interpreting this concrete UserSecretReference. + * Should be of the form: + * 'urn:polaris-secret:<secret-manager-type>:<type-specific-identifier> + * @param referencePayload Optionally, any additional information that is necessary to fully + * reconstitute the original secret based on what is retrieved by the {@code urn}; this + * payload may include hashes/checksums, encryption key ids, OTP encryption keys, additional + * protocol/version specifiers, etc., which are implementation-specific. + */ + public UserSecretReference( + @JsonProperty(value = "urn", required = true) @Nonnull String urn, + @JsonProperty(value = "referencePayload") @Nullable Map referencePayload) { + // TODO: Add better/standardized parsing and validation of URN syntax + Preconditions.checkArgument( + urn.startsWith("urn:polaris-secret:") && urn.split(":").length >= 4, + "Invalid secret URN '%s'; must be of the form " + + "'urn:polaris-secret::'", + urn); + this.urn = urn; + this.referencePayload = Objects.requireNonNullElse(referencePayload, new HashMap<>()); + } + + /** + * Since UserSecretReference objects are specific to UserSecretManager implementations, the + * "secret-manager-type" portion of the URN should be used to validate that a URN is valid for a + * given implementation and to dispatch to the correct implementation at runtime if multiple + * concurrent implementations are possible in a given runtime environment. + */ + @JsonIgnore + public String getUserSecretManagerTypeFromUrn() { + // TODO: Add better/standardized parsing and validation of URN syntax + return urn.split(":")[2]; + } + + public @Nonnull String getUrn() { + return urn; + } + + public @Nonnull Map getReferencePayload() { + return referencePayload; + } + + @Override + public int hashCode() { + return Objects.hash(getUrn(), getReferencePayload()); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || !(obj instanceof UserSecretReference)) { + return false; + } + UserSecretReference that = (UserSecretReference) obj; + return Objects.equals(this.getUrn(), that.getUrn()) + && Objects.equals(this.getReferencePayload(), that.getReferencePayload()); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("urn", getUrn()) + .add("referencePayload", String.format("", getReferencePayload().size())) + .toString(); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManager.java b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManager.java new file mode 100644 index 0000000000..b1418efc9a --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManager.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.secrets; + +import jakarta.annotation.Nonnull; +import org.apache.polaris.core.entity.PolarisEntityCore; + +/** + * Manages secrets specified by users of the Polaris API, either directly or as an intermediary + * layer between Polaris and external secret-management systems. Such secrets are distinct from + * "service-level" secrets that pertain to the Polaris service itself which would be more statically + * configured system-wide. In contrast, user-owned secrets are handled dynamically as part of + * runtime API requests. + */ +public interface UserSecretsManager { + /** + * Persist the {@code secret} under a new URN {@code secretUrn} and return a {@code + * UserSecretReference} that can subsequently be used by this same UserSecretsManager to retrieve + * the original secret. The {@code forEntity} is provided for an implementation to extract other + * identifying metadata such as entity type, id, name, etc., to store alongside the remotely + * stored secret to facilitate operational management of the secrets outside of the core Polaris + * service (for example, to perform garbage-collection if the Polaris service fails to delete + * managed secrets in the external system when associated entities are deleted). + * + * @param secret The secret to store + * @param forEntity The PolarisEntity that is associated with the secret + * @return A reference object that can be used to retrieve the secret which is safe to store in + * its entirety within a persisted PolarisEntity + */ + @Nonnull + UserSecretReference writeSecret(@Nonnull String secret, @Nonnull PolarisEntityCore forEntity); + + /** + * Retrieve a secret using the {@code secretReference}. See {@link UserSecretReference} for + * details about identifiers and payloads. + * + * @param secretReference Reference object for retrieving the original secret + * @return The stored secret, or null if it no longer exists + */ + @Nonnull + String readSecret(@Nonnull UserSecretReference secretReference); + + /** + * Delete a stored secret. See {@link UserSecretReference} for details about identifiers and + * payloads. + * + * @param secretReference Reference object for retrieving the original secret + */ + void deleteSecret(@Nonnull UserSecretReference secretReference); +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManagerFactory.java new file mode 100644 index 0000000000..a8ee2b7b44 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManagerFactory.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.secrets; + +import org.apache.polaris.core.context.RealmContext; + +/** + * Factory for creating {@link UserSecretsManager} instances. + * + *

Each {@link UserSecretsManager} instance is associated with a {@link RealmContext} and is + * responsible for managing the secrets for the user in that realm. + */ +public interface UserSecretsManagerFactory { + UserSecretsManager getOrCreateUserSecretsManager(RealmContext realmContext); +} diff --git a/polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java b/polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java new file mode 100644 index 0000000000..7be1e6ed29 --- /dev/null +++ b/polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.connection; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.admin.model.ConnectionConfigInfo; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class ConnectionConfigInfoDpoTest { + PolarisDiagnostics polarisDiagnostics = new PolarisDefaultDiagServiceImpl(); + ObjectMapper objectMapper = new ObjectMapper(); + + @Test + void testOAuthClientCredentialsParameters() throws JsonProcessingException { + // Test deserialization and reserialization of the persistence JSON. + String json = + "" + + "{" + + " \"connectionTypeCode\": 1," + + " \"uri\": \"https://myorg-my_account.snowflakecomputing.com/polaris/api/catalog\"," + + " \"remoteCatalogName\": \"my-catalog\"," + + " \"authenticationParameters\": {" + + " \"authenticationTypeCode\": 1," + + " \"tokenUri\": \"https://myorg-my_account.snowflakecomputing.com/polaris/api/catalog/v1/oauth/tokens\"," + + " \"clientId\": \"client-id\"," + + " \"clientSecretReference\": {" + + " \"urn\": \"urn:polaris-secret:secretmanager-impl:keystore-id-12345\"," + + " \"referencePayload\": {" + + " \"hash\": \"a1b2c3\"," + + " \"encryption-key\": \"z0y9x8\"" + + " }" + + " }," + + " \"scopes\": [\"PRINCIPAL_ROLE:ALL\"]" + + " }" + + "}"; + ConnectionConfigInfoDpo connectionConfigInfoDpo = + ConnectionConfigInfoDpo.deserialize(polarisDiagnostics, json); + Assertions.assertNotNull(connectionConfigInfoDpo); + JsonNode tree1 = objectMapper.readTree(json); + JsonNode tree2 = objectMapper.readTree(connectionConfigInfoDpo.serialize()); + Assertions.assertEquals(tree1, tree2); + + // Test conversion into API model JSON. + ConnectionConfigInfo connectionConfigInfoApiModel = + connectionConfigInfoDpo.asConnectionConfigInfoModel(); + String expectedApiModelJson = + "" + + "{" + + " \"connectionType\": \"ICEBERG_REST\"," + + " \"uri\": \"https://myorg-my_account.snowflakecomputing.com/polaris/api/catalog\"," + + " \"remoteCatalogName\": \"my-catalog\"," + + " \"authenticationParameters\": {" + + " \"authenticationType\": \"OAUTH\"," + + " \"tokenUri\": \"https://myorg-my_account.snowflakecomputing.com/polaris/api/catalog/v1/oauth/tokens\"," + + " \"clientId\": \"client-id\"," + + " \"scopes\": [\"PRINCIPAL_ROLE:ALL\"]" + + " }" + + "}"; + Assertions.assertEquals( + objectMapper.readValue(expectedApiModelJson, ConnectionConfigInfo.class), + connectionConfigInfoApiModel); + } + + @Test + void testBearerAuthenticationParameters() throws JsonProcessingException { + // Test deserialization and reserialization of the persistence JSON. + String json = + "" + + "{" + + " \"connectionTypeCode\": 1," + + " \"uri\": \"https://myorg-my_account.snowflakecomputing.com/polaris/api/catalog\"," + + " \"remoteCatalogName\": \"my-catalog\"," + + " \"authenticationParameters\": {" + + " \"authenticationTypeCode\": 2," + + " \"bearerTokenReference\": {" + + " \"urn\": \"urn:polaris-secret:secretmanager-impl:keystore-id-12345\"," + + " \"referencePayload\": {" + + " \"hash\": \"a1b2c3\"," + + " \"encryption-key\": \"z0y9x8\"" + + " }" + + " }" + + " }" + + "}"; + ConnectionConfigInfoDpo connectionConfigInfoDpo = + ConnectionConfigInfoDpo.deserialize(polarisDiagnostics, json); + Assertions.assertNotNull(connectionConfigInfoDpo); + JsonNode tree1 = objectMapper.readTree(json); + JsonNode tree2 = objectMapper.readTree(connectionConfigInfoDpo.serialize()); + Assertions.assertEquals(tree1, tree2); + + // Test conversion into API model JSON. + ConnectionConfigInfo connectionConfigInfoApiModel = + connectionConfigInfoDpo.asConnectionConfigInfoModel(); + String expectedApiModelJson = + "" + + "{" + + " \"connectionType\": \"ICEBERG_REST\"," + + " \"uri\": \"https://myorg-my_account.snowflakecomputing.com/polaris/api/catalog\"," + + " \"remoteCatalogName\": \"my-catalog\"," + + " \"authenticationParameters\": {" + + " \"authenticationType\": \"BEARER\"" + + " }" + + "}"; + Assertions.assertEquals( + objectMapper.readValue(expectedApiModelJson, ConnectionConfigInfo.class), + connectionConfigInfoApiModel); + } +} diff --git a/polaris-core/src/test/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManagerTest.java b/polaris-core/src/test/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManagerTest.java new file mode 100644 index 0000000000..bac1fd6491 --- /dev/null +++ b/polaris-core/src/test/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManagerTest.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.secrets; + +class UnsafeInMemorySecretsManagerTest extends UserSecretsManagerBaseTest { + @Override + protected UserSecretsManager newSecretsManager() { + return new UnsafeInMemorySecretsManager(); + } +} diff --git a/polaris-core/src/test/java/org/apache/polaris/core/secrets/UserSecretsManagerBaseTest.java b/polaris-core/src/test/java/org/apache/polaris/core/secrets/UserSecretsManagerBaseTest.java new file mode 100644 index 0000000000..09e45b185a --- /dev/null +++ b/polaris-core/src/test/java/org/apache/polaris/core/secrets/UserSecretsManagerBaseTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.secrets; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.core.entity.PolarisEntity; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Base test class for implementations of UserSecretsManager which can be extended by different + * implementation-specific unittests. + */ +public abstract class UserSecretsManagerBaseTest { + private static final ObjectMapper DEFAULT_MAPPER = new ObjectMapper(); + + /** + * @return a fresh instance of a UserSecretsManager to use in test cases. + */ + protected abstract UserSecretsManager newSecretsManager(); + + @Test + public void testBasicSecretStorageAndRetrieval() throws JsonProcessingException { + UserSecretsManager secretsManager = newSecretsManager(); + + PolarisEntity entity1 = new CatalogEntity.Builder().setId(1111L).setName("entity1").build(); + PolarisEntity entity2 = new CatalogEntity.Builder().setId(2222L).setName("entity2").build(); + + String secret1 = "sensitivesecret1"; + String secret2 = "sensitivesecret2"; + + UserSecretReference reference1 = secretsManager.writeSecret(secret1, entity1); + UserSecretReference reference2 = secretsManager.writeSecret(secret2, entity2); + + // Make sure we can JSON-serialize and deserialize the UserSecretReference objects. + String serializedReference1 = DEFAULT_MAPPER.writeValueAsString(reference1); + String serializedReference2 = DEFAULT_MAPPER.writeValueAsString(reference2); + + UserSecretReference reassembledReference1 = + DEFAULT_MAPPER.readValue(serializedReference1, UserSecretReference.class); + UserSecretReference reassembledReference2 = + DEFAULT_MAPPER.readValue(serializedReference2, UserSecretReference.class); + + Assertions.assertThat(reassembledReference1).isEqualTo(reference1); + Assertions.assertThat(reassembledReference2).isEqualTo(reference2); + Assertions.assertThat(secretsManager.readSecret(reassembledReference1)).isEqualTo(secret1); + Assertions.assertThat(secretsManager.readSecret(reassembledReference2)).isEqualTo(secret2); + } + + @Test + public void testMultipleSecretsForSameEntity() { + UserSecretsManager secretsManager = newSecretsManager(); + + PolarisEntity entity1 = new CatalogEntity.Builder().setId(1111L).setName("entity1").build(); + + String secret1 = "sensitivesecret1"; + String secret2 = "sensitivesecret2"; + + UserSecretReference reference1 = secretsManager.writeSecret(secret1, entity1); + UserSecretReference reference2 = secretsManager.writeSecret(secret2, entity1); + + Assertions.assertThat(secretsManager.readSecret(reference1)).isEqualTo(secret1); + Assertions.assertThat(secretsManager.readSecret(reference2)).isEqualTo(secret2); + } + + @Test + public void testDeleteSecret() { + UserSecretsManager secretsManager = newSecretsManager(); + + PolarisEntity entity1 = new CatalogEntity.Builder().setId(1111L).setName("entity1").build(); + + String secret1 = "sensitivesecret1"; + + UserSecretReference reference1 = secretsManager.writeSecret(secret1, entity1); + + Assertions.assertThat(secretsManager.readSecret(reference1)).isEqualTo(secret1); + + secretsManager.deleteSecret(reference1); + Assertions.assertThat(secretsManager.readSecret(reference1)) + .as("Deleted secret should return null") + .isNull(); + } +} diff --git a/quarkus/defaults/src/main/resources/application-it.properties b/quarkus/defaults/src/main/resources/application-it.properties index 5e110071de..4419d4d590 100644 --- a/quarkus/defaults/src/main/resources/application-it.properties +++ b/quarkus/defaults/src/main/resources/application-it.properties @@ -29,6 +29,8 @@ quarkus.management.port=0 # polaris.persistence.type=in-memory-atomic polaris.persistence.type=in-memory +polaris.secrets-manager.type=in-memory + polaris.features.defaults."ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING"=false polaris.features.defaults."ALLOW_EXTERNAL_METADATA_FILE_LOCATION"=false polaris.features.defaults."ALLOW_OVERLAPPING_CATALOG_URLS"=true @@ -38,6 +40,7 @@ polaris.features.defaults."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKI polaris.features.defaults."INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_it"=true polaris.features.defaults."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"=true polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES"=["FILE","S3","GCS","AZURE"] +polaris.features.defaults."ENABLE_CATALOG_FEDERATION"=true polaris.realm-context.realms=POLARIS,OTHER diff --git a/quarkus/defaults/src/main/resources/application.properties b/quarkus/defaults/src/main/resources/application.properties index 0cd79eadae..c34c1bc0e9 100644 --- a/quarkus/defaults/src/main/resources/application.properties +++ b/quarkus/defaults/src/main/resources/application.properties @@ -91,6 +91,8 @@ polaris.realm-context.require-header=false polaris.features.defaults."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"=false polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE","FILE"] +# polaris.features.defaults."ENABLE_CATALOG_FEDERATION"=true + # realm overrides # polaris.features.realm-overrides."my-realm"."INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST"=true # polaris.features.realm-overrides."my-realm"."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"=true @@ -99,6 +101,8 @@ polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE", # polaris.persistence.type=in-memory-atomic polaris.persistence.type=in-memory +polaris.secrets-manager.type=in-memory + polaris.file-io.type=default polaris.log.request-id-header-name=Polaris-Request-Id diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java index ffab9a9412..625e57e45d 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java @@ -46,6 +46,8 @@ import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.apache.polaris.core.secrets.UserSecretsManager; +import org.apache.polaris.core.secrets.UserSecretsManagerFactory; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.auth.ActiveRolesProvider; import org.apache.polaris.service.auth.Authenticator; @@ -62,6 +64,7 @@ import org.apache.polaris.service.quarkus.persistence.QuarkusPersistenceConfiguration; import org.apache.polaris.service.quarkus.ratelimiter.QuarkusRateLimiterFilterConfiguration; import org.apache.polaris.service.quarkus.ratelimiter.QuarkusTokenBucketConfiguration; +import org.apache.polaris.service.quarkus.secrets.QuarkusSecretsManagerConfiguration; import org.apache.polaris.service.ratelimiter.RateLimiter; import org.apache.polaris.service.ratelimiter.TokenBucketFactory; import org.apache.polaris.service.task.TaskHandlerConfiguration; @@ -150,6 +153,13 @@ public MetaStoreManagerFactory metaStoreManagerFactory( return metaStoreManagerFactories.select(Identifier.Literal.of(config.type())).get(); } + @Produces + public UserSecretsManagerFactory userSecretsManagerFactory( + QuarkusSecretsManagerConfiguration config, + @Any Instance userSecretsManagerFactories) { + return userSecretsManagerFactories.select(Identifier.Literal.of(config.type())).get(); + } + /** * Eagerly initialize the in-memory default realm on startup, so that users can check the * credentials printed to stdout immediately. @@ -220,6 +230,13 @@ public PolarisMetaStoreManager polarisMetaStoreManager( return metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); } + @Produces + @RequestScoped + public UserSecretsManager userSecretsManager( + RealmContext realmContext, UserSecretsManagerFactory userSecretsManagerFactory) { + return userSecretsManagerFactory.getOrCreateUserSecretsManager(realmContext); + } + @Produces @RequestScoped public BasePersistence polarisMetaStoreSession( diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/secrets/QuarkusSecretsManagerConfiguration.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/secrets/QuarkusSecretsManagerConfiguration.java new file mode 100644 index 0000000000..a472dc681f --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/secrets/QuarkusSecretsManagerConfiguration.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.secrets; + +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; + +@StaticInitSafe +@ConfigMapping(prefix = "polaris.secrets-manager") +public interface QuarkusSecretsManagerConfiguration { + + /** + * The type of the UserSecretsManagerFactory to use. This is the {@link + * org.apache.polaris.core.secrets.UserSecretsManagerFactory} identifier. + */ + String type(); +} diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAdminServiceAuthzTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAdminServiceAuthzTest.java index 5c055eec47..55b2d9f341 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAdminServiceAuthzTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAdminServiceAuthzTest.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Function; +import org.apache.polaris.core.admin.model.CreateCatalogRequest; import org.apache.polaris.core.admin.model.UpdateCatalogRequest; import org.apache.polaris.core.admin.model.UpdateCatalogRoleRequest; import org.apache.polaris.core.admin.model.UpdatePrincipalRequest; @@ -52,6 +53,7 @@ private PolarisAdminService newTestAdminService(Set activatedPrincipalRo callContext, entityManager, metaStoreManager, + userSecretsManager, securityContext(authenticatedPrincipal, activatedPrincipalRoles), polarisAuthorizer); } @@ -130,13 +132,14 @@ public void testCreateCatalogSufficientPrivileges() { PRINCIPAL_ROLE2, PolarisPrivilege.CATALOG_DROP)) .isTrue(); final CatalogEntity newCatalog = new CatalogEntity.Builder().setName("new_catalog").build(); + final CreateCatalogRequest createRequest = new CreateCatalogRequest(newCatalog.asCatalog()); doTestSufficientPrivileges( List.of( PolarisPrivilege.SERVICE_MANAGE_ACCESS, PolarisPrivilege.CATALOG_CREATE, PolarisPrivilege.CATALOG_FULL_METADATA), - () -> newTestAdminService(Set.of(PRINCIPAL_ROLE1)).createCatalog(newCatalog), + () -> newTestAdminService(Set.of(PRINCIPAL_ROLE1)).createCatalog(createRequest), () -> newTestAdminService(Set.of(PRINCIPAL_ROLE2)).deleteCatalog(newCatalog.getName()), (privilege) -> adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), @@ -148,6 +151,7 @@ public void testCreateCatalogSufficientPrivileges() { @Test public void testCreateCatalogInsufficientPrivileges() { final CatalogEntity newCatalog = new CatalogEntity.Builder().setName("new_catalog").build(); + final CreateCatalogRequest createRequest = new CreateCatalogRequest(newCatalog.asCatalog()); doTestInsufficientPrivileges( List.of( @@ -164,7 +168,7 @@ public void testCreateCatalogInsufficientPrivileges() { PolarisPrivilege.CATALOG_MANAGE_METADATA, PolarisPrivilege.CATALOG_MANAGE_CONTENT, PolarisPrivilege.CATALOG_MANAGE_ACCESS), - () -> newTestAdminService(Set.of(PRINCIPAL_ROLE1)).createCatalog(newCatalog), + () -> newTestAdminService(Set.of(PRINCIPAL_ROLE1)).createCatalog(createRequest), (privilege) -> adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), (privilege) -> @@ -283,7 +287,8 @@ public void testDeleteCatalogSufficientPrivileges() { PRINCIPAL_ROLE2, PolarisPrivilege.CATALOG_CREATE)) .isTrue(); final CatalogEntity newCatalog = new CatalogEntity.Builder().setName("new_catalog").build(); - adminService.createCatalog(newCatalog); + final CreateCatalogRequest createRequest = new CreateCatalogRequest(newCatalog.asCatalog()); + adminService.createCatalog(createRequest); doTestSufficientPrivileges( List.of( @@ -291,7 +296,7 @@ public void testDeleteCatalogSufficientPrivileges() { PolarisPrivilege.CATALOG_DROP, PolarisPrivilege.CATALOG_FULL_METADATA), () -> newTestAdminService(Set.of(PRINCIPAL_ROLE1)).deleteCatalog(newCatalog.getName()), - () -> newTestAdminService(Set.of(PRINCIPAL_ROLE2)).createCatalog(newCatalog), + () -> newTestAdminService(Set.of(PRINCIPAL_ROLE2)).createCatalog(createRequest), (privilege) -> adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), (privilege) -> @@ -302,7 +307,8 @@ public void testDeleteCatalogSufficientPrivileges() { @Test public void testDeleteCatalogInsufficientPrivileges() { final CatalogEntity newCatalog = new CatalogEntity.Builder().setName("new_catalog").build(); - adminService.createCatalog(newCatalog); + final CreateCatalogRequest createRequest = new CreateCatalogRequest(newCatalog.asCatalog()); + adminService.createCatalog(createRequest); doTestInsufficientPrivileges( List.of( diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java index 61a26b1c7b..4b0037eed0 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java @@ -48,6 +48,7 @@ import org.apache.iceberg.types.Types; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.admin.model.CreateCatalogRequest; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.PrincipalWithCredentials; import org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials; @@ -74,6 +75,8 @@ import org.apache.polaris.core.persistence.dao.entity.EntityResult; import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; +import org.apache.polaris.core.secrets.UserSecretsManager; +import org.apache.polaris.core.secrets.UserSecretsManagerFactory; import org.apache.polaris.service.admin.PolarisAdminService; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; import org.apache.polaris.service.catalog.generic.GenericTableCatalog; @@ -175,6 +178,7 @@ public Map getConfigOverrides() { @Inject protected MetaStoreManagerFactory managerFactory; @Inject protected RealmEntityManagerFactory realmEntityManagerFactory; @Inject protected CallContextCatalogFactory callContextCatalogFactory; + @Inject protected UserSecretsManagerFactory userSecretsManagerFactory; @Inject protected PolarisDiagnostics diagServices; @Inject protected Clock clock; @Inject protected FileIOFactory fileIOFactory; @@ -184,6 +188,7 @@ public Map getConfigOverrides() { protected PolarisAdminService adminService; protected PolarisEntityManager entityManager; protected PolarisMetaStoreManager metaStoreManager; + protected UserSecretsManager userSecretsManager; protected TransactionalPersistence metaStoreSession; protected PolarisBaseEntity catalogEntity; protected PrincipalEntity principalEntity; @@ -204,6 +209,7 @@ public static void setUpMocks() { public void before(TestInfo testInfo) { RealmContext realmContext = testInfo::getDisplayName; metaStoreManager = managerFactory.getOrCreateMetaStoreManager(realmContext); + userSecretsManager = userSecretsManagerFactory.getOrCreateUserSecretsManager(realmContext); Map configMap = Map.of( @@ -247,6 +253,7 @@ public void before(TestInfo testInfo) { callContext, entityManager, metaStoreManager, + userSecretsManager, securityContext(authenticatedRoot, Set.of()), polarisAuthorizer); @@ -258,12 +265,14 @@ public void before(TestInfo testInfo) { .build(); catalogEntity = adminService.createCatalog( - new CatalogEntity.Builder() - .setName(CATALOG_NAME) - .setCatalogType("INTERNAL") - .setDefaultBaseLocation(storageLocation) - .setStorageConfigurationInfo(storageConfigModel, storageLocation) - .build()); + new CreateCatalogRequest( + new CatalogEntity.Builder() + .setName(CATALOG_NAME) + .setCatalogType("INTERNAL") + .setDefaultBaseLocation(storageLocation) + .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .build() + .asCatalog())); initBaseCatalog(); @@ -462,16 +471,22 @@ public static class TestPolarisCallContextCatalogFactory extends PolarisCallContextCatalogFactory { public TestPolarisCallContextCatalogFactory() { - super(null, null, null, null); + super(null, null, null, null, null); } @Inject public TestPolarisCallContextCatalogFactory( RealmEntityManagerFactory entityManagerFactory, MetaStoreManagerFactory metaStoreManagerFactory, + UserSecretsManagerFactory userSecretsManagerFactory, TaskExecutor taskExecutor, FileIOFactory fileIOFactory) { - super(entityManagerFactory, metaStoreManagerFactory, taskExecutor, fileIOFactory); + super( + entityManagerFactory, + metaStoreManagerFactory, + userSecretsManagerFactory, + taskExecutor, + fileIOFactory); } @Override diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java index ddcdf075d3..462cc7bb73 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java @@ -45,6 +45,7 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; +import org.apache.polaris.core.admin.model.CreateCatalogRequest; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.auth.PolarisAuthorizerImpl; @@ -66,6 +67,8 @@ import org.apache.polaris.core.persistence.dao.entity.BaseResult; import org.apache.polaris.core.persistence.dao.entity.PrincipalSecretsResult; import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; +import org.apache.polaris.core.secrets.UserSecretsManager; +import org.apache.polaris.core.secrets.UserSecretsManagerFactory; import org.apache.polaris.core.storage.PolarisStorageIntegration; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; import org.apache.polaris.core.storage.aws.AwsCredentialsStorageIntegration; @@ -118,6 +121,7 @@ public Map getConfigOverrides() { public static final String SESSION_TOKEN = "session_token"; @Inject MetaStoreManagerFactory managerFactory; + @Inject UserSecretsManagerFactory userSecretsManagerFactory; @Inject PolarisConfigurationStore configurationStore; @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; @Inject PolarisDiagnostics diagServices; @@ -128,6 +132,7 @@ public Map getConfigOverrides() { private AwsStorageConfigInfo storageConfigModel; private String realmName; private PolarisMetaStoreManager metaStoreManager; + private UserSecretsManager userSecretsManager; private PolarisCallContext polarisContext; private PolarisAdminService adminService; private PolarisEntityManager entityManager; @@ -157,6 +162,7 @@ public void before(TestInfo testInfo) { testInfo.getTestMethod().map(Method::getName).orElse("test"), System.nanoTime()); RealmContext realmContext = () -> realmName; metaStoreManager = managerFactory.getOrCreateMetaStoreManager(realmContext); + userSecretsManager = userSecretsManagerFactory.getOrCreateUserSecretsManager(realmContext); polarisContext = new PolarisCallContext( managerFactory.getOrCreateSessionSupplier(realmContext).get(), @@ -191,6 +197,7 @@ public void before(TestInfo testInfo) { callContext, entityManager, metaStoreManager, + userSecretsManager, securityContext, new PolarisAuthorizerImpl(new PolarisConfigurationStore() {})); @@ -205,16 +212,19 @@ public void before(TestInfo testInfo) { .build(); catalogEntity = adminService.createCatalog( - new CatalogEntity.Builder() - .setName(CATALOG_NAME) - .setDefaultBaseLocation(storageLocation) - .setReplaceNewLocationPrefixWithCatalogDefault("file:") - .addProperty( - FeatureConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") - .addProperty( - FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .setStorageConfigurationInfo(storageConfigModel, storageLocation) - .build()); + new CreateCatalogRequest( + new CatalogEntity.Builder() + .setName(CATALOG_NAME) + .setDefaultBaseLocation(storageLocation) + .setReplaceNewLocationPrefixWithCatalogDefault("file:") + .addProperty( + FeatureConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") + .addProperty( + FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), + "true") + .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .build() + .asCatalog())); PolarisPassthroughResolutionView passthroughView = new PolarisPassthroughResolutionView( diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java index 207955c932..11a938f316 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java @@ -50,6 +50,7 @@ import org.apache.iceberg.rest.requests.UpdateTableRequest; import org.apache.iceberg.view.ImmutableSQLViewRepresentation; import org.apache.iceberg.view.ImmutableViewVersion; +import org.apache.polaris.core.admin.model.CreateCatalogRequest; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials; import org.apache.polaris.core.admin.model.StorageConfigInfo; @@ -110,6 +111,7 @@ private IcebergCatalogHandler newWrapper( callContext, entityManager, metaStoreManager, + userSecretsManager, securityContext(authenticatedPrincipal, activatedPrincipalRoles), factory, catalogName, @@ -248,6 +250,7 @@ public void testInsufficientPermissionsPriorToSecretRotation() { callContext, entityManager, metaStoreManager, + userSecretsManager, securityContext(authenticatedPrincipal, Set.of(PRINCIPAL_ROLE1, PRINCIPAL_ROLE2)), callContextCatalogFactory, CATALOG_NAME, @@ -280,6 +283,7 @@ public void testInsufficientPermissionsPriorToSecretRotation() { callContext, entityManager, metaStoreManager, + userSecretsManager, securityContext(authenticatedPrincipal1, Set.of(PRINCIPAL_ROLE1, PRINCIPAL_ROLE2)), callContextCatalogFactory, CATALOG_NAME, @@ -1724,12 +1728,14 @@ public void testSendNotificationSufficientPrivileges() { .setStorageType(StorageConfigInfo.StorageTypeEnum.FILE) .build(); adminService.createCatalog( - new CatalogEntity.Builder() - .setName(externalCatalog) - .setDefaultBaseLocation(storageLocation) - .setStorageConfigurationInfo(storageConfigModel, storageLocation) - .setCatalogType("EXTERNAL") - .build()); + new CreateCatalogRequest( + new CatalogEntity.Builder() + .setName(externalCatalog) + .setDefaultBaseLocation(storageLocation) + .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .setCatalogType("EXTERNAL") + .build() + .asCatalog())); adminService.createCatalogRole( externalCatalog, new CatalogRoleEntity.Builder().setName(CATALOG_ROLE1).build()); adminService.createCatalogRole( @@ -1799,6 +1805,7 @@ public PolarisEntityManager getOrCreateEntityManager(RealmContext realmContext) } }, managerFactory, + userSecretsManagerFactory, Mockito.mock(), new DefaultFileIOFactory( realmEntityManagerFactory, managerFactory, new PolarisConfigurationStore() {})) { diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java index 4a5506fa22..690a335a2b 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java @@ -74,6 +74,7 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; +import org.apache.polaris.core.admin.model.CreateCatalogRequest; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.auth.PolarisAuthorizerImpl; @@ -98,6 +99,8 @@ import org.apache.polaris.core.persistence.dao.entity.EntityResult; import org.apache.polaris.core.persistence.dao.entity.PrincipalSecretsResult; import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; +import org.apache.polaris.core.secrets.UserSecretsManager; +import org.apache.polaris.core.secrets.UserSecretsManagerFactory; import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.core.storage.PolarisStorageIntegration; @@ -172,12 +175,14 @@ public Map getConfigOverrides() { @Inject MetaStoreManagerFactory managerFactory; @Inject PolarisConfigurationStore configurationStore; @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; + @Inject UserSecretsManagerFactory userSecretsManagerFactory; @Inject PolarisDiagnostics diagServices; private IcebergCatalog catalog; private CallContext callContext; private String realmName; private PolarisMetaStoreManager metaStoreManager; + private UserSecretsManager userSecretsManager; private PolarisCallContext polarisContext; private PolarisAdminService adminService; private PolarisEntityManager entityManager; @@ -204,6 +209,7 @@ public void before(TestInfo testInfo) { testInfo.getTestMethod().map(Method::getName).orElse("test"), System.nanoTime()); RealmContext realmContext = () -> realmName; metaStoreManager = managerFactory.getOrCreateMetaStoreManager(realmContext); + userSecretsManager = userSecretsManagerFactory.getOrCreateUserSecretsManager(realmContext); polarisContext = new PolarisCallContext( managerFactory.getOrCreateSessionSupplier(realmContext).get(), @@ -239,6 +245,7 @@ public void before(TestInfo testInfo) { callContext, entityManager, metaStoreManager, + userSecretsManager, securityContext, new PolarisAuthorizerImpl(new PolarisConfigurationStore() {})); @@ -253,16 +260,19 @@ public void before(TestInfo testInfo) { .build(); catalogEntity = adminService.createCatalog( - new CatalogEntity.Builder() - .setName(CATALOG_NAME) - .setDefaultBaseLocation(storageLocation) - .setReplaceNewLocationPrefixWithCatalogDefault("file:") - .addProperty( - FeatureConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") - .addProperty( - FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .setStorageConfigurationInfo(storageConfigModel, storageLocation) - .build()); + new CreateCatalogRequest( + new CatalogEntity.Builder() + .setName(CATALOG_NAME) + .setDefaultBaseLocation(storageLocation) + .setReplaceNewLocationPrefixWithCatalogDefault("file:") + .addProperty( + FeatureConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") + .addProperty( + FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), + "true") + .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .build() + .asCatalog())); RealmEntityManagerFactory realmEntityManagerFactory = new RealmEntityManagerFactory(createMockMetaStoreManagerFactory()); @@ -893,10 +903,12 @@ public void testUpdateNotificationCreateTableWithLocalFilePrefix() { String catalogWithoutStorage = "catalogWithoutStorage"; PolarisEntity catalogEntity = adminService.createCatalog( - new CatalogEntity.Builder() - .setDefaultBaseLocation("file://") - .setName(catalogWithoutStorage) - .build()); + new CreateCatalogRequest( + new CatalogEntity.Builder() + .setDefaultBaseLocation("file://") + .setName(catalogWithoutStorage) + .build() + .asCatalog())); PolarisPassthroughResolutionView passthroughView = new PolarisPassthroughResolutionView( @@ -957,10 +969,12 @@ public void testUpdateNotificationCreateTableWithHttpPrefix() { String catalogName = "catalogForMaliciousDomain"; adminService.createCatalog( - new CatalogEntity.Builder() - .setDefaultBaseLocation("http://maliciousdomain.com") - .setName(catalogName) - .build()); + new CreateCatalogRequest( + new CatalogEntity.Builder() + .setDefaultBaseLocation("http://maliciousdomain.com") + .setName(catalogName) + .build() + .asCatalog())); PolarisPassthroughResolutionView passthroughView = new PolarisPassthroughResolutionView( @@ -1496,16 +1510,19 @@ public void testDropTableWithPurgeDisabled() { .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) .build(); adminService.createCatalog( - new CatalogEntity.Builder() - .setName(noPurgeCatalogName) - .setDefaultBaseLocation(storageLocation) - .setReplaceNewLocationPrefixWithCatalogDefault("file:") - .addProperty(FeatureConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") - .addProperty( - FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .addProperty(FeatureConfiguration.DROP_WITH_PURGE_ENABLED.catalogConfig(), "false") - .setStorageConfigurationInfo(noPurgeStorageConfigModel, storageLocation) - .build()); + new CreateCatalogRequest( + new CatalogEntity.Builder() + .setName(noPurgeCatalogName) + .setDefaultBaseLocation(storageLocation) + .setReplaceNewLocationPrefixWithCatalogDefault("file:") + .addProperty( + FeatureConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") + .addProperty( + FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") + .addProperty(FeatureConfiguration.DROP_WITH_PURGE_ENABLED.catalogConfig(), "false") + .setStorageConfigurationInfo(noPurgeStorageConfigModel, storageLocation) + .build() + .asCatalog())); PolarisPassthroughResolutionView passthroughView = new PolarisPassthroughResolutionView( callContext, entityManager, securityContext, noPurgeCatalogName); diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java index 7d817bec4f..8d7708dd73 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java @@ -40,6 +40,7 @@ import org.apache.iceberg.view.ViewCatalogTests; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.admin.model.CreateCatalogRequest; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; @@ -57,6 +58,8 @@ import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.cache.EntityCache; +import org.apache.polaris.core.secrets.UserSecretsManager; +import org.apache.polaris.core.secrets.UserSecretsManagerFactory; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.admin.PolarisAdminService; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; @@ -97,6 +100,7 @@ public Map getConfigOverrides() { public static final String CATALOG_NAME = "polaris-catalog"; @Inject MetaStoreManagerFactory managerFactory; + @Inject UserSecretsManagerFactory userSecretsManagerFactory; @Inject PolarisConfigurationStore configurationStore; @Inject PolarisDiagnostics diagServices; @@ -104,6 +108,7 @@ public Map getConfigOverrides() { private String realmName; private PolarisMetaStoreManager metaStoreManager; + private UserSecretsManager userSecretsManager; private PolarisCallContext polarisContext; @BeforeAll @@ -130,6 +135,7 @@ public void before(TestInfo testInfo) { RealmContext realmContext = () -> realmName; metaStoreManager = managerFactory.getOrCreateMetaStoreManager(realmContext); + userSecretsManager = userSecretsManagerFactory.getOrCreateUserSecretsManager(realmContext); polarisContext = new PolarisCallContext( managerFactory.getOrCreateSessionSupplier(realmContext).get(), @@ -166,20 +172,24 @@ public void before(TestInfo testInfo) { callContext, entityManager, metaStoreManager, + userSecretsManager, securityContext, new PolarisAuthorizerImpl(new PolarisConfigurationStore() {})); adminService.createCatalog( - new CatalogEntity.Builder() - .setName(CATALOG_NAME) - .addProperty(FeatureConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") - .addProperty( - FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .setDefaultBaseLocation("file://tmp") - .setStorageConfigurationInfo( - new FileStorageConfigInfo( - StorageConfigInfo.StorageTypeEnum.FILE, List.of("file://", "/", "*")), - "file://tmp") - .build()); + new CreateCatalogRequest( + new CatalogEntity.Builder() + .setName(CATALOG_NAME) + .addProperty( + FeatureConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") + .addProperty( + FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") + .setDefaultBaseLocation("file://tmp") + .setStorageConfigurationInfo( + new FileStorageConfigInfo( + StorageConfigInfo.StorageTypeEnum.FILE, List.of("file://", "/", "*")), + "file://tmp") + .build() + .asCatalog())); PolarisPassthroughResolutionView passthroughView = new PolarisPassthroughResolutionView( diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java index 9e4d4b38a6..e2c3be8916 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java @@ -52,6 +52,7 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; +import org.apache.polaris.core.admin.model.CreateCatalogRequest; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.auth.PolarisAuthorizerImpl; @@ -77,6 +78,8 @@ import org.apache.polaris.core.policy.exceptions.NoSuchPolicyException; import org.apache.polaris.core.policy.exceptions.PolicyVersionMismatchException; import org.apache.polaris.core.policy.validator.InvalidPolicyException; +import org.apache.polaris.core.secrets.UserSecretsManager; +import org.apache.polaris.core.secrets.UserSecretsManagerFactory; import org.apache.polaris.core.storage.PolarisStorageIntegration; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; import org.apache.polaris.core.storage.aws.AwsCredentialsStorageIntegration; @@ -144,6 +147,7 @@ public Map getConfigOverrides() { PolicyAttachmentTarget.TypeEnum.TABLE_LIKE, List.of(TABLE.toString().split("\\."))); @Inject MetaStoreManagerFactory managerFactory; + @Inject UserSecretsManagerFactory userSecretsManagerFactory; @Inject PolarisConfigurationStore configurationStore; @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; @Inject PolarisDiagnostics diagServices; @@ -154,6 +158,7 @@ public Map getConfigOverrides() { private AwsStorageConfigInfo storageConfigModel; private String realmName; private PolarisMetaStoreManager metaStoreManager; + private UserSecretsManager userSecretsManager; private PolarisCallContext polarisContext; private PolarisAdminService adminService; private PolarisEntityManager entityManager; @@ -178,6 +183,7 @@ public void before(TestInfo testInfo) { testInfo.getTestMethod().map(Method::getName).orElse("test"), System.nanoTime()); RealmContext realmContext = () -> realmName; metaStoreManager = managerFactory.getOrCreateMetaStoreManager(realmContext); + userSecretsManager = userSecretsManagerFactory.getOrCreateUserSecretsManager(realmContext); polarisContext = new PolarisCallContext( managerFactory.getOrCreateSessionSupplier(realmContext).get(), @@ -212,6 +218,7 @@ public void before(TestInfo testInfo) { callContext, entityManager, metaStoreManager, + userSecretsManager, securityContext, new PolarisAuthorizerImpl(new PolarisConfigurationStore() {})); @@ -226,16 +233,19 @@ public void before(TestInfo testInfo) { .build(); catalogEntity = adminService.createCatalog( - new CatalogEntity.Builder() - .setName(CATALOG_NAME) - .setDefaultBaseLocation(storageLocation) - .setReplaceNewLocationPrefixWithCatalogDefault("file:") - .addProperty( - FeatureConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") - .addProperty( - FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .setStorageConfigurationInfo(storageConfigModel, storageLocation) - .build()); + new CreateCatalogRequest( + new CatalogEntity.Builder() + .setName(CATALOG_NAME) + .setDefaultBaseLocation(storageLocation) + .setReplaceNewLocationPrefixWithCatalogDefault("file:") + .addProperty( + FeatureConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") + .addProperty( + FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), + "true") + .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .build() + .asCatalog())); PolarisPassthroughResolutionView passthroughView = new PolarisPassthroughResolutionView( diff --git a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java index 8f1c9bc920..9b40f4228f 100644 --- a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java +++ b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java @@ -26,6 +26,7 @@ import jakarta.ws.rs.core.SecurityContext; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -44,11 +45,18 @@ import org.apache.iceberg.exceptions.ValidationException; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.admin.model.AuthenticationParameters; +import org.apache.polaris.core.admin.model.BearerAuthenticationParameters; +import org.apache.polaris.core.admin.model.Catalog; import org.apache.polaris.core.admin.model.CatalogGrant; import org.apache.polaris.core.admin.model.CatalogPrivilege; +import org.apache.polaris.core.admin.model.ConnectionConfigInfo; +import org.apache.polaris.core.admin.model.CreateCatalogRequest; +import org.apache.polaris.core.admin.model.ExternalCatalog; import org.apache.polaris.core.admin.model.GrantResource; import org.apache.polaris.core.admin.model.NamespaceGrant; import org.apache.polaris.core.admin.model.NamespacePrivilege; +import org.apache.polaris.core.admin.model.OAuthClientCredentialsParameters; import org.apache.polaris.core.admin.model.PrincipalWithCredentials; import org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials; import org.apache.polaris.core.admin.model.TableGrant; @@ -64,6 +72,7 @@ import org.apache.polaris.core.auth.PolarisAuthorizer; import org.apache.polaris.core.catalog.PolarisCatalogHelpers; import org.apache.polaris.core.config.FeatureConfiguration; +import org.apache.polaris.core.connection.AuthenticationParametersDpo; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.CatalogRoleEntity; @@ -89,6 +98,8 @@ import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; import org.apache.polaris.core.persistence.resolver.ResolverPath; import org.apache.polaris.core.persistence.resolver.ResolverStatus; +import org.apache.polaris.core.secrets.UserSecretReference; +import org.apache.polaris.core.secrets.UserSecretsManager; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.StorageLocation; import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; @@ -116,6 +127,7 @@ public class PolarisAdminService { private final AuthenticatedPolarisPrincipal authenticatedPrincipal; private final PolarisAuthorizer authorizer; private final PolarisMetaStoreManager metaStoreManager; + private final UserSecretsManager userSecretsManager; // Initialized in the authorize methods. private PolarisResolutionManifest resolutionManifest = null; @@ -124,6 +136,7 @@ public PolarisAdminService( @NotNull CallContext callContext, @NotNull PolarisEntityManager entityManager, @NotNull PolarisMetaStoreManager metaStoreManager, + @NotNull UserSecretsManager userSecretsManager, @NotNull SecurityContext securityContext, @NotNull PolarisAuthorizer authorizer) { this.callContext = callContext; @@ -141,12 +154,17 @@ public PolarisAdminService( this.authenticatedPrincipal = (AuthenticatedPolarisPrincipal) securityContext.getUserPrincipal(); this.authorizer = authorizer; + this.userSecretsManager = userSecretsManager; } private PolarisCallContext getCurrentPolarisContext() { return callContext.getPolarisCallContext(); } + private UserSecretsManager getUserSecretsManager() { + return userSecretsManager; + } + private Optional findCatalogByName(String name) { return Optional.ofNullable(resolutionManifest.getResolvedReferenceCatalogEntity()) .map(path -> CatalogEntity.of(path.getRawLeafEntity())); @@ -559,10 +577,87 @@ private boolean catalogOverlapsWithExistingCatalog(CatalogEntity catalogEntity) }); } - public PolarisEntity createCatalog(PolarisEntity entity) { + /** + * Secrets embedded *or* simply referenced through the API model will require separate processing + * for normalizing into resolved/verified/offloaded UserSecretReference objects which are then + * placed appropriately into persistence objects. + * + *

If secrets are already direct URIs/URNs to an external secret store, we may need to validate + * the URI/URN and/or transform into a polaris-internal URN format along with type-information or + * other secrets-manager metadata in the referencePayload. + * + *

If secrets reference first-class Polaris-stored secrets, we must resolve the associated + * polaris persistence entities defining access to those secrets and perform authorization. + * + *

If secrets are provided inline as part of the request, we must explicitly offload the + * secrets into a Polaris service-level secrets manager and return the associated internal + * references to the stored secret. + */ + private Map extractSecretReferences( + CreateCatalogRequest catalogRequest, PolarisEntity forEntity) { + Map secretReferences = new HashMap<>(); + Catalog catalog = catalogRequest.getCatalog(); + UserSecretsManager secretsManager = getUserSecretsManager(); + if (catalog instanceof ExternalCatalog externalCatalog) { + if (externalCatalog.getConnectionConfigInfo() != null) { + ConnectionConfigInfo connectionConfig = externalCatalog.getConnectionConfigInfo(); + AuthenticationParameters authenticationParameters = + connectionConfig.getAuthenticationParameters(); + + switch (authenticationParameters.getAuthenticationType()) { + case OAUTH: + { + OAuthClientCredentialsParameters oauthClientCredentialsModel = + (OAuthClientCredentialsParameters) authenticationParameters; + String inlineClientSecret = oauthClientCredentialsModel.getClientSecret(); + UserSecretReference secretReference = + secretsManager.writeSecret(inlineClientSecret, forEntity); + secretReferences.put( + AuthenticationParametersDpo.INLINE_CLIENT_SECRET_REFERENCE_KEY, secretReference); + break; + } + case BEARER: + { + BearerAuthenticationParameters bearerAuthenticationParametersModel = + (BearerAuthenticationParameters) authenticationParameters; + String inlineBearerToken = bearerAuthenticationParametersModel.getBearerToken(); + UserSecretReference secretReference = + secretsManager.writeSecret(inlineBearerToken, forEntity); + secretReferences.put( + AuthenticationParametersDpo.INLINE_BEARER_TOKEN_REFERENCE_KEY, secretReference); + break; + } + default: + throw new IllegalStateException( + "Unsupported authentication type: " + + authenticationParameters.getAuthenticationType()); + } + } + } + return secretReferences; + } + + /** + * @see #extractSecretReferences + */ + private boolean requiresSecretReferenceExtraction(CreateCatalogRequest catalogRequest) { + Catalog catalog = catalogRequest.getCatalog(); + if (catalog instanceof ExternalCatalog externalCatalog) { + if (externalCatalog.getConnectionConfigInfo() != null) { + // TODO: Make this more targeted once we have connection configs that don't involve + // processing of inline secrets. + return true; + } + } + return false; + } + + public PolarisEntity createCatalog(CreateCatalogRequest catalogRequest) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.CREATE_CATALOG; authorizeBasicRootOperationOrThrow(op); + CatalogEntity entity = CatalogEntity.fromCatalog(catalogRequest.getCatalog()); + checkArgument(entity.getId() == -1, "Entity to be created must have no ID assigned"); if (catalogOverlapsWithExistingCatalog((CatalogEntity) entity)) { @@ -571,14 +666,38 @@ public PolarisEntity createCatalog(PolarisEntity entity) { entity.getName()); } - PolarisEntity polarisEntity = - new PolarisEntity.Builder(entity) + // After basic validations, now populate id and creation timestamp. + entity = + new CatalogEntity.Builder(entity) .setId(metaStoreManager.generateNewEntityId(getCurrentPolarisContext()).getId()) .setCreateTimestamp(System.currentTimeMillis()) .build(); + + if (requiresSecretReferenceExtraction(catalogRequest)) { + LOGGER + .atDebug() + .addKeyValue("catalogName", entity.getName()) + .log("Extracting secret references to create federated catalog"); + FeatureConfiguration.enforceFeatureEnabledOrThrow( + callContext, FeatureConfiguration.ENABLE_CATALOG_FEDERATION); + // For fields that contain references to secrets, we'll separately process the secrets from + // the original request first, and then populate those fields with the extracted secret + // references as part of the construction of the internal persistence entity. + Map processedSecretReferences = + extractSecretReferences(catalogRequest, entity); + entity = + new CatalogEntity.Builder(entity) + .setConnectionConfigInfoDpoWithSecrets( + ((ExternalCatalog) catalogRequest.getCatalog()).getConnectionConfigInfo(), + processedSecretReferences) + .build(); + } + CreateCatalogResult catalogResult = - metaStoreManager.createCatalog(getCurrentPolarisContext(), polarisEntity, List.of()); + metaStoreManager.createCatalog(getCurrentPolarisContext(), entity, List.of()); if (catalogResult.alreadyExists()) { + // TODO: Proactive garbage-collection of any inline secrets that were written to the + // secrets manager, here and on any other unexpected exception as well. throw new AlreadyExistsException( "Cannot create Catalog %s. Catalog already exists or resolution failed", entity.getName()); diff --git a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java index 8b92956f8e..c2c53c17b2 100644 --- a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java +++ b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java @@ -68,6 +68,8 @@ import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.secrets.UserSecretsManager; +import org.apache.polaris.core.secrets.UserSecretsManagerFactory; import org.apache.polaris.service.admin.api.PolarisCatalogsApiService; import org.apache.polaris.service.admin.api.PolarisPrincipalRolesApiService; import org.apache.polaris.service.admin.api.PolarisPrincipalsApiService; @@ -85,16 +87,19 @@ public class PolarisServiceImpl private final RealmEntityManagerFactory entityManagerFactory; private final PolarisAuthorizer polarisAuthorizer; private final MetaStoreManagerFactory metaStoreManagerFactory; + private final UserSecretsManagerFactory userSecretsManagerFactory; private final CallContext callContext; @Inject public PolarisServiceImpl( RealmEntityManagerFactory entityManagerFactory, MetaStoreManagerFactory metaStoreManagerFactory, + UserSecretsManagerFactory userSecretsManagerFactory, PolarisAuthorizer polarisAuthorizer, CallContext callContext) { this.entityManagerFactory = entityManagerFactory; this.metaStoreManagerFactory = metaStoreManagerFactory; + this.userSecretsManagerFactory = userSecretsManagerFactory; this.polarisAuthorizer = polarisAuthorizer; this.callContext = callContext; // FIXME: This is a hack to set the current context for downstream calls. @@ -113,8 +118,15 @@ private PolarisAdminService newAdminService( entityManagerFactory.getOrCreateEntityManager(realmContext); PolarisMetaStoreManager metaStoreManager = metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); + UserSecretsManager userSecretsManager = + userSecretsManagerFactory.getOrCreateUserSecretsManager(realmContext); return new PolarisAdminService( - callContext, entityManager, metaStoreManager, securityContext, polarisAuthorizer); + callContext, + entityManager, + metaStoreManager, + userSecretsManager, + securityContext, + polarisAuthorizer); } /** From PolarisCatalogsApiService */ @@ -124,9 +136,7 @@ public Response createCatalog( PolarisAdminService adminService = newAdminService(realmContext, securityContext); Catalog catalog = request.getCatalog(); validateStorageConfig(catalog.getStorageConfigInfo()); - Catalog newCatalog = - new CatalogEntity(adminService.createCatalog(CatalogEntity.fromCatalog(catalog))) - .asCatalog(); + Catalog newCatalog = new CatalogEntity(adminService.createCatalog(request)).asCatalog(); LOGGER.info("Created new catalog {}", newCatalog); return Response.status(Response.Status.CREATED).build(); } diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java index 8d7b49a071..6895351395 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java @@ -66,6 +66,7 @@ import org.apache.polaris.core.persistence.resolver.Resolver; import org.apache.polaris.core.persistence.resolver.ResolverStatus; import org.apache.polaris.core.rest.PolarisEndpoints; +import org.apache.polaris.core.secrets.UserSecretsManager; import org.apache.polaris.service.catalog.AccessDelegationMode; import org.apache.polaris.service.catalog.CatalogPrefixParser; import org.apache.polaris.service.catalog.api.IcebergRestCatalogApiService; @@ -132,6 +133,7 @@ public class IcebergCatalogAdapter private final CallContextCatalogFactory catalogFactory; private final PolarisEntityManager entityManager; private final PolarisMetaStoreManager metaStoreManager; + private final UserSecretsManager userSecretsManager; private final PolarisAuthorizer polarisAuthorizer; private final CatalogPrefixParser prefixParser; @@ -142,6 +144,7 @@ public IcebergCatalogAdapter( CallContextCatalogFactory catalogFactory, PolarisEntityManager entityManager, PolarisMetaStoreManager metaStoreManager, + UserSecretsManager userSecretsManager, PolarisAuthorizer polarisAuthorizer, CatalogPrefixParser prefixParser) { this.realmContext = realmContext; @@ -149,6 +152,7 @@ public IcebergCatalogAdapter( this.catalogFactory = catalogFactory; this.entityManager = entityManager; this.metaStoreManager = metaStoreManager; + this.userSecretsManager = userSecretsManager; this.polarisAuthorizer = polarisAuthorizer; this.prefixParser = prefixParser; @@ -188,6 +192,7 @@ private IcebergCatalogHandler newHandlerWrapper( callContext, entityManager, metaStoreManager, + userSecretsManager, securityContext, catalogFactory, catalogName, diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index ace46a3a30..716d68fdbc 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -43,6 +43,7 @@ import org.apache.iceberg.UpdateRequirement; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.SessionCatalog; import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.catalog.ViewCatalog; @@ -52,6 +53,8 @@ import org.apache.iceberg.exceptions.ForbiddenException; import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.rest.CatalogHandlers; +import org.apache.iceberg.rest.HTTPClient; +import org.apache.iceberg.rest.RESTCatalog; import org.apache.iceberg.rest.credentials.ImmutableCredential; import org.apache.iceberg.rest.requests.CommitTransactionRequest; import org.apache.iceberg.rest.requests.CreateNamespaceRequest; @@ -72,6 +75,9 @@ import org.apache.polaris.core.auth.PolarisAuthorizer; import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.config.PolarisConfigurationStore; +import org.apache.polaris.core.connection.ConnectionConfigInfoDpo; +import org.apache.polaris.core.connection.ConnectionType; +import org.apache.polaris.core.connection.IcebergRestConnectionConfigInfoDpo; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisEntitySubType; @@ -82,6 +88,7 @@ import org.apache.polaris.core.persistence.TransactionWorkspaceMetaStoreManager; import org.apache.polaris.core.persistence.dao.entity.EntitiesResult; import org.apache.polaris.core.persistence.dao.entity.EntityWithPath; +import org.apache.polaris.core.secrets.UserSecretsManager; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.service.catalog.SupportsNotifications; import org.apache.polaris.service.catalog.common.CatalogHandler; @@ -111,6 +118,7 @@ public class IcebergCatalogHandler extends CatalogHandler implements AutoCloseab private static final Logger LOGGER = LoggerFactory.getLogger(IcebergCatalogHandler.class); private final PolarisMetaStoreManager metaStoreManager; + private final UserSecretsManager userSecretsManager; private final CallContextCatalogFactory catalogFactory; // Catalog instance will be initialized after authorizing resolver successfully resolves @@ -123,25 +131,17 @@ public IcebergCatalogHandler( CallContext callContext, PolarisEntityManager entityManager, PolarisMetaStoreManager metaStoreManager, + UserSecretsManager userSecretsManager, SecurityContext securityContext, CallContextCatalogFactory catalogFactory, String catalogName, PolarisAuthorizer authorizer) { super(callContext, entityManager, securityContext, catalogName, authorizer); this.metaStoreManager = metaStoreManager; + this.userSecretsManager = userSecretsManager; this.catalogFactory = catalogFactory; } - @Override - protected void initializeCatalog() { - this.baseCatalog = - catalogFactory.createCallContextCatalog( - callContext, authenticatedPrincipal, securityContext, resolutionManifest); - this.namespaceCatalog = - (baseCatalog instanceof SupportsNamespaces) ? (SupportsNamespaces) baseCatalog : null; - this.viewCatalog = (baseCatalog instanceof ViewCatalog) ? (ViewCatalog) baseCatalog : null; - } - /** * TODO: Make the helper in org.apache.iceberg.rest.CatalogHandlers public instead of needing to * copy/paste here. @@ -163,6 +163,57 @@ public static boolean isCreate(UpdateTableRequest request) { return isCreate; } + private UserSecretsManager getUserSecretsManager() { + return userSecretsManager; + } + + @Override + protected void initializeCatalog() { + CatalogEntity resolvedCatalogEntity = + CatalogEntity.of(resolutionManifest.getResolvedReferenceCatalogEntity().getRawLeafEntity()); + ConnectionConfigInfoDpo connectionConfigInfoDpo = + resolvedCatalogEntity.getConnectionConfigInfoDpo(); + if (connectionConfigInfoDpo != null) { + LOGGER + .atInfo() + .addKeyValue("remoteUrl", connectionConfigInfoDpo.getUri()) + .log("Initializing federated catalog"); + FeatureConfiguration.enforceFeatureEnabledOrThrow( + callContext, FeatureConfiguration.ENABLE_CATALOG_FEDERATION); + + Catalog federatedCatalog; + ConnectionType connectionType = + ConnectionType.fromCode(connectionConfigInfoDpo.getConnectionTypeCode()); + switch (connectionType) { + case ICEBERG_REST: + SessionCatalog.SessionContext context = SessionCatalog.SessionContext.createEmpty(); + federatedCatalog = + new RESTCatalog( + context, + (config) -> + HTTPClient.builder(config) + .uri(config.get(org.apache.iceberg.CatalogProperties.URI)) + .build()); + federatedCatalog.initialize( + ((IcebergRestConnectionConfigInfoDpo) connectionConfigInfoDpo).getRemoteCatalogName(), + connectionConfigInfoDpo.asIcebergCatalogProperties(getUserSecretsManager())); + break; + default: + throw new UnsupportedOperationException( + "Connection type not supported: " + connectionType); + } + this.baseCatalog = federatedCatalog; + } else { + LOGGER.atInfo().log("Initializing non-federated catalog"); + this.baseCatalog = + catalogFactory.createCallContextCatalog( + callContext, authenticatedPrincipal, securityContext, resolutionManifest); + } + this.namespaceCatalog = + (baseCatalog instanceof SupportsNamespaces) ? (SupportsNamespaces) baseCatalog : null; + this.viewCatalog = (baseCatalog instanceof ViewCatalog) ? (ViewCatalog) baseCatalog : null; + } + public ListNamespacesResponse listNamespaces(Namespace parent) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LIST_NAMESPACES; authorizeBasicNamespaceOperationOrThrow(op, parent); @@ -204,9 +255,10 @@ public CreateNamespaceResponse createNamespace(CreateNamespaceRequest request) { } } - private static boolean isExternal(CatalogEntity catalog) { + private static boolean isStaticFacade(CatalogEntity catalog) { return org.apache.polaris.core.admin.model.Catalog.TypeEnum.EXTERNAL.equals( - catalog.getCatalogType()); + catalog.getCatalogType()) + && !catalog.isPassthroughFacade(); } public GetNamespaceResponse loadNamespaceMetadata(Namespace namespace) { @@ -271,8 +323,8 @@ public LoadTableResponse createTableDirect(Namespace namespace, CreateTableReque .getResolvedReferenceCatalogEntity() .getResolvedLeafEntity() .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot create table on external catalogs."); + if (isStaticFacade(catalog)) { + throw new BadRequestException("Cannot create table on static-facade external catalogs."); } return CatalogHandlers.createTable(baseCatalog, namespace, request); } @@ -297,8 +349,8 @@ public LoadTableResponse createTableDirectWithWriteDelegation( .getResolvedReferenceCatalogEntity() .getResolvedLeafEntity() .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot create table on external catalogs."); + if (isStaticFacade(catalog)) { + throw new BadRequestException("Cannot create table on static-facade external catalogs."); } request.validate(); @@ -392,8 +444,8 @@ public LoadTableResponse createTableStaged(Namespace namespace, CreateTableReque .getResolvedReferenceCatalogEntity() .getResolvedLeafEntity() .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot create table on external catalogs."); + if (isStaticFacade(catalog)) { + throw new BadRequestException("Cannot create table on static-facade external catalogs."); } TableMetadata metadata = stageTableCreateHelper(namespace, request); return LoadTableResponse.builder().withTableMetadata(metadata).build(); @@ -412,8 +464,8 @@ public LoadTableResponse createTableStagedWithWriteDelegation( .getResolvedReferenceCatalogEntity() .getResolvedLeafEntity() .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot create table on external catalogs."); + if (isStaticFacade(catalog)) { + throw new BadRequestException("Cannot create table on static-facade external catalogs."); } TableIdentifier ident = TableIdentifier.of(namespace, request.name()); TableMetadata metadata = stageTableCreateHelper(namespace, request); @@ -699,8 +751,8 @@ public LoadTableResponse updateTable( .getResolvedReferenceCatalogEntity() .getResolvedLeafEntity() .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot update table on external catalogs."); + if (isStaticFacade(catalog)) { + throw new BadRequestException("Cannot update table on static-facade external catalogs."); } return CatalogHandlers.updateTable(baseCatalog, tableIdentifier, applyUpdateFilters(request)); } @@ -716,8 +768,8 @@ public LoadTableResponse updateTableForStagedCreate( .getResolvedReferenceCatalogEntity() .getResolvedLeafEntity() .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot update table on external catalogs."); + if (isStaticFacade(catalog)) { + throw new BadRequestException("Cannot update table on static-facade external catalogs."); } return CatalogHandlers.updateTable(baseCatalog, tableIdentifier, applyUpdateFilters(request)); } @@ -741,8 +793,8 @@ public void dropTableWithPurge(TableIdentifier tableIdentifier) { .getResolvedReferenceCatalogEntity() .getResolvedLeafEntity() .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot drop table on external catalogs."); + if (isStaticFacade(catalog)) { + throw new BadRequestException("Cannot drop table on static-facade external catalogs."); } CatalogHandlers.purgeTable(baseCatalog, tableIdentifier); } @@ -767,8 +819,8 @@ public void renameTable(RenameTableRequest request) { .getResolvedReferenceCatalogEntity() .getResolvedLeafEntity() .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot rename table on external catalogs."); + if (isStaticFacade(catalog)) { + throw new BadRequestException("Cannot rename table on static-facade external catalogs."); } CatalogHandlers.renameTable(baseCatalog, request); } @@ -791,8 +843,8 @@ public void commitTransaction(CommitTransactionRequest commitTransactionRequest) .getResolvedReferenceCatalogEntity() .getResolvedLeafEntity() .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot update table on external catalogs."); + if (isStaticFacade(catalog)) { + throw new BadRequestException("Cannot update table on static-facade external catalogs."); } if (!(baseCatalog instanceof IcebergCatalog)) { @@ -897,8 +949,8 @@ public LoadViewResponse createView(Namespace namespace, CreateViewRequest reques .getResolvedReferenceCatalogEntity() .getResolvedLeafEntity() .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot create view on external catalogs."); + if (isStaticFacade(catalog)) { + throw new BadRequestException("Cannot create view on static-facade external catalogs."); } return CatalogHandlers.createView(viewCatalog, namespace, request); } @@ -920,8 +972,8 @@ public LoadViewResponse replaceView(TableIdentifier viewIdentifier, UpdateTableR .getResolvedReferenceCatalogEntity() .getResolvedLeafEntity() .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot replace view on external catalogs."); + if (isStaticFacade(catalog)) { + throw new BadRequestException("Cannot replace view on static-facade external catalogs."); } return CatalogHandlers.updateView(viewCatalog, viewIdentifier, applyUpdateFilters(request)); } @@ -952,8 +1004,8 @@ public void renameView(RenameTableRequest request) { .getResolvedReferenceCatalogEntity() .getResolvedLeafEntity() .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot rename view on external catalogs."); + if (isStaticFacade(catalog)) { + throw new BadRequestException("Cannot rename view on static-facade external catalogs."); } CatalogHandlers.renameView(viewCatalog, request); } diff --git a/service/common/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java b/service/common/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java index 0f4df51107..94fe197609 100644 --- a/service/common/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java @@ -34,6 +34,7 @@ import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; +import org.apache.polaris.core.secrets.UserSecretsManagerFactory; import org.apache.polaris.service.catalog.iceberg.IcebergCatalog; import org.apache.polaris.service.catalog.io.FileIOFactory; import org.apache.polaris.service.config.RealmEntityManagerFactory; @@ -53,15 +54,18 @@ public class PolarisCallContextCatalogFactory implements CallContextCatalogFacto private final TaskExecutor taskExecutor; private final FileIOFactory fileIOFactory; private final MetaStoreManagerFactory metaStoreManagerFactory; + private final UserSecretsManagerFactory userSecretsManagerFactory; @Inject public PolarisCallContextCatalogFactory( RealmEntityManagerFactory entityManagerFactory, MetaStoreManagerFactory metaStoreManagerFactory, + UserSecretsManagerFactory userSecretsManagerFactory, TaskExecutor taskExecutor, FileIOFactory fileIOFactory) { this.entityManagerFactory = entityManagerFactory; this.metaStoreManagerFactory = metaStoreManagerFactory; + this.userSecretsManagerFactory = userSecretsManagerFactory; this.taskExecutor = taskExecutor; this.fileIOFactory = fileIOFactory; } diff --git a/service/common/src/main/java/org/apache/polaris/service/secrets/UnsafeInMemorySecretsManagerFactory.java b/service/common/src/main/java/org/apache/polaris/service/secrets/UnsafeInMemorySecretsManagerFactory.java new file mode 100644 index 0000000000..fb11b9acd7 --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/secrets/UnsafeInMemorySecretsManagerFactory.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.secrets; + +import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.secrets.UnsafeInMemorySecretsManager; +import org.apache.polaris.core.secrets.UserSecretsManager; +import org.apache.polaris.core.secrets.UserSecretsManagerFactory; + +@ApplicationScoped +@Identifier("in-memory") +public class UnsafeInMemorySecretsManagerFactory implements UserSecretsManagerFactory { + private final Map cachedSecretsManagers = new ConcurrentHashMap<>(); + + @Override + public UserSecretsManager getOrCreateUserSecretsManager(RealmContext realmContext) { + return cachedSecretsManagers.computeIfAbsent( + realmContext.getRealmIdentifier(), key -> new UnsafeInMemorySecretsManager()); + } +} diff --git a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java index bfe97d8ca5..aa357208a5 100644 --- a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java +++ b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java @@ -42,6 +42,8 @@ import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.dao.entity.CreatePrincipalResult; import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; +import org.apache.polaris.core.secrets.UserSecretsManager; +import org.apache.polaris.core.secrets.UserSecretsManagerFactory; import org.apache.polaris.service.admin.PolarisServiceImpl; import org.apache.polaris.service.admin.api.PolarisCatalogsApi; import org.apache.polaris.service.catalog.DefaultCatalogPrefixParser; @@ -55,6 +57,7 @@ import org.apache.polaris.service.context.CallContextCatalogFactory; import org.apache.polaris.service.context.PolarisCallContextCatalogFactory; import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; +import org.apache.polaris.service.secrets.UnsafeInMemorySecretsManagerFactory; import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; import org.apache.polaris.service.task.TaskExecutor; import org.assertj.core.util.TriFunction; @@ -132,6 +135,8 @@ public TestServices build() { storageIntegrationProvider, polarisDiagnostics); RealmEntityManagerFactory realmEntityManagerFactory = new RealmEntityManagerFactory(metaStoreManagerFactory) {}; + UserSecretsManagerFactory userSecretsManagerFactory = + new UnsafeInMemorySecretsManagerFactory(); TransactionalPersistence metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); @@ -161,6 +166,8 @@ public Map contextVariables() { realmEntityManagerFactory.getOrCreateEntityManager(realmContext); PolarisMetaStoreManager metaStoreManager = metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); + UserSecretsManager userSecretsManager = + userSecretsManagerFactory.getOrCreateUserSecretsManager(realmContext); FileIOFactory fileIOFactory = fileIOFactorySupplier.apply( @@ -170,7 +177,11 @@ public Map contextVariables() { CallContextCatalogFactory callContextFactory = new PolarisCallContextCatalogFactory( - realmEntityManagerFactory, metaStoreManagerFactory, taskExecutor, fileIOFactory); + realmEntityManagerFactory, + metaStoreManagerFactory, + userSecretsManagerFactory, + taskExecutor, + fileIOFactory); IcebergCatalogAdapter service = new IcebergCatalogAdapter( @@ -179,6 +190,7 @@ public Map contextVariables() { callContextFactory, entityManager, metaStoreManager, + userSecretsManager, authorizer, new DefaultCatalogPrefixParser()); @@ -223,7 +235,11 @@ public String getAuthenticationScheme() { PolarisCatalogsApi catalogsApi = new PolarisCatalogsApi( new PolarisServiceImpl( - realmEntityManagerFactory, metaStoreManagerFactory, authorizer, callContext)); + realmEntityManagerFactory, + metaStoreManagerFactory, + userSecretsManagerFactory, + authorizer, + callContext)); return new TestServices( catalogsApi,