diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java index efa2441a1f..d647893255 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java @@ -334,7 +334,7 @@ public void testIcebergCreateNamespaceInExternalCatalog() throws IOException { .isNotNull() .isNotEmpty() .containsEntry( - PolarisEntityConstants.ENTITY_BASE_LOCATION, "s3://my-bucket/path/to/data/db1"); + PolarisEntityConstants.ENTITY_BASE_LOCATION, "s3://my-bucket/path/to/data/db1/"); } } diff --git a/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java index 02997f37ce..b7b0a951e6 100644 --- a/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java +++ b/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java @@ -32,6 +32,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; import java.util.function.Predicate; @@ -41,9 +42,11 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.EntityNameLookupRecord; +import org.apache.polaris.core.entity.LocationBasedEntity; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; +import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; import org.apache.polaris.core.entity.PolarisEntityType; @@ -778,4 +781,12 @@ public void rollback() { session.getTransaction().rollback(); } } + + /** {@inheritDoc} */ + @Override + public + Optional> hasOverlappingSiblings( + @Nonnull PolarisCallContext callContext, T entity) { + return Optional.empty(); + } } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java index b26247aad0..b0f5dcd8c7 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java @@ -44,6 +44,6 @@ public static DatabaseType fromDisplayName(String displayName) { } public String getInitScriptResource() { - return String.format("%s/schema-v1.sql", this.getDisplayName()); + return String.format("%s/schema-v2.sql", this.getDisplayName()); } } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java index 2fb0c90ca0..5c3dd1dbaf 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java @@ -31,13 +31,16 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.entity.EntityNameLookupRecord; +import org.apache.polaris.core.entity.LocationBasedEntity; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; +import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; import org.apache.polaris.core.entity.PolarisEntityType; @@ -59,10 +62,12 @@ import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageIntegration; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; +import org.apache.polaris.core.storage.StorageLocation; import org.apache.polaris.persistence.relational.jdbc.models.ModelEntity; import org.apache.polaris.persistence.relational.jdbc.models.ModelGrantRecord; import org.apache.polaris.persistence.relational.jdbc.models.ModelPolicyMappingRecord; import org.apache.polaris.persistence.relational.jdbc.models.ModelPrincipalAuthenticationData; +import org.apache.polaris.persistence.relational.jdbc.models.SchemaVersion; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,6 +79,10 @@ public class JdbcBasePersistenceImpl implements BasePersistence, IntegrationPers private final PrincipalSecretsGenerator secretsGenerator; private final PolarisStorageIntegrationProvider storageIntegrationProvider; private final String realmId; + private final int version; + + // The max number of components a location can have before the optimized sibling check is not used + private static final int MAX_LOCATION_COMPONENTS = 40; public JdbcBasePersistenceImpl( DatasourceOperations databaseOperations, @@ -84,6 +93,7 @@ public JdbcBasePersistenceImpl( this.secretsGenerator = secretsGenerator; this.storageIntegrationProvider = storageIntegrationProvider; this.realmId = realmId; + this.version = loadVersion(); } @Override @@ -622,6 +632,64 @@ public boolean hasChildren( } } + private int loadVersion() { + PreparedQuery query = QueryGenerator.generateVersionQuery(); + try { + List schemaVersion = + datasourceOperations.executeSelect(query, new SchemaVersion()); + if (schemaVersion == null || schemaVersion.size() != 1) { + throw new RuntimeException("Failed to retrieve schema version"); + } + return schemaVersion.getFirst().getValue(); + } catch (SQLException e) { + LOGGER.error("Failed to load schema version due to {}", e.getMessage(), e); + throw new IllegalStateException("Failed to retrieve schema version", e); + } + } + + /** {@inheritDoc} */ + @Override + public + Optional> hasOverlappingSiblings( + @Nonnull PolarisCallContext callContext, T entity) { + if (this.version < 2) { + return Optional.empty(); + } + if (entity.getBaseLocation().chars().filter(ch -> ch == '/').count() + > MAX_LOCATION_COMPONENTS) { + return Optional.empty(); + } + + PreparedQuery query = + QueryGenerator.generateOverlapQuery( + realmId, entity.getCatalogId(), entity.getBaseLocation()); + try { + var results = datasourceOperations.executeSelect(query, new ModelEntity()); + if (!results.isEmpty()) { + StorageLocation entityLocation = StorageLocation.of(entity.getBaseLocation()); + for (PolarisBaseEntity result : results) { + StorageLocation potentialSiblingLocation = + StorageLocation.of(((LocationBasedEntity) result).getBaseLocation()); + if (entityLocation.isChildOf(potentialSiblingLocation) + || potentialSiblingLocation.isChildOf(entityLocation)) { + return Optional.of(Optional.of(potentialSiblingLocation.toString())); + } + } + } + return Optional.of(Optional.empty()); + } catch (SQLException e) { + LOGGER.error( + "Failed to retrieve location overlap for location {} due to {}", + entity.getBaseLocation(), + e.getMessage(), + e); + throw new RuntimeException( + String.format( + "Failed to retrieve location overlap for location: %s", entity.getBaseLocation()), + e); + } + } + @Nullable @Override public PolarisPrincipalSecrets loadPrincipalSecrets( diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java index ecd5b2783c..0e7ea56307 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java @@ -30,6 +30,7 @@ import java.util.stream.Collectors; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; +import org.apache.polaris.core.storage.StorageLocation; import org.apache.polaris.persistence.relational.jdbc.models.ModelEntity; import org.apache.polaris.persistence.relational.jdbc.models.ModelGrantRecord; @@ -208,6 +209,59 @@ static QueryFragment generateWhereClause( return new QueryFragment(clause, parameters); } + @VisibleForTesting + static PreparedQuery generateVersionQuery() { + return new PreparedQuery("SELECT version_value FROM POLARIS_SCHEMA.VERSION", List.of()); + } + + /** + * Generate a SELECT query to find any entities that have a given realm & parent and that may with + * a given location. The check is performed without consideration for the scheme, so a path on one + * storage type may give a false positive for overlapping with another storage type. This should + * be combined with a check using `StorageLocation`. + * + * @param realmId A realm to search within + * @param catalogId A catalog entity to search within + * @param baseLocation The base location to look for overlap with, with or without a scheme + * @return The list of possibly overlapping entities that meet the criteria + */ + @VisibleForTesting + public static PreparedQuery generateOverlapQuery( + String realmId, long catalogId, String baseLocation) { + StorageLocation baseStorageLocation = StorageLocation.of(baseLocation); + String locationWithoutScheme = baseStorageLocation.withoutScheme(); + + List conditions = new ArrayList<>(); + List parameters = new ArrayList<>(); + + String[] components = locationWithoutScheme.split("/"); + StringBuilder pathBuilder = new StringBuilder(); + + for (String component : components) { + pathBuilder.append(component).append("/"); + conditions.add("location_without_scheme = ?"); + parameters.add(pathBuilder.toString()); + } + + // Add LIKE condition to match children + conditions.add("location_without_scheme LIKE ?"); + parameters.add(locationWithoutScheme + "%"); + + String locationClause = String.join(" OR ", conditions); + String clause = " WHERE realm_id = ? AND catalog_id = ? AND (" + locationClause + ")"; + + // realmId and parentId go first + List finalParams = new ArrayList<>(); + finalParams.add(realmId); + finalParams.add(catalogId); + finalParams.addAll(parameters); + + QueryFragment where = new QueryFragment(clause, finalParams); + PreparedQuery query = + generateSelectQuery(ModelEntity.ALL_COLUMNS, ModelEntity.TABLE_NAME, where.sql()); + return new PreparedQuery(query.sql(), where.parameters()); + } + private static String getFullyQualifiedTableName(String tableName) { // TODO: make schema name configurable. return "POLARIS_SCHEMA." + tableName; diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java index b847a677f2..5504171400 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java @@ -24,8 +24,10 @@ import java.util.List; import java.util.Map; import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.storage.StorageLocation; import org.apache.polaris.persistence.relational.jdbc.DatabaseType; public class ModelEntity implements Converter { @@ -47,7 +49,8 @@ public class ModelEntity implements Converter { "last_update_timestamp", "properties", "internal_properties", - "grant_records_version"); + "grant_records_version", + "location_without_scheme"); // the id of the catalog associated to that entity. use 0 if this entity is top-level // like a catalog @@ -95,6 +98,9 @@ public class ModelEntity implements Converter { // current version for that entity, will be monotonically incremented private int grantRecordsVersion; + // location for the entity but without a scheme, when applicable + private String locationWithoutScheme; + public long getId() { return id; } @@ -155,6 +161,10 @@ public int getGrantRecordsVersion() { return grantRecordsVersion; } + public String getLocationWithoutScheme() { + return locationWithoutScheme; + } + public static Builder builder() { return new Builder(); } @@ -180,6 +190,7 @@ public PolarisBaseEntity fromResultSet(ResultSet r) throws SQLException { // JSONB: use getString(), not getObject(). .internalProperties(r.getString("internal_properties")) .grantRecordsVersion(r.getObject("grant_records_version", Integer.class)) + .locationWithoutScheme(r.getString("location_without_scheme")) .build(); return toEntity(modelEntity); @@ -208,6 +219,7 @@ public Map toMap(DatabaseType databaseType) { map.put("internal_properties", this.getInternalProperties()); } map.put("grant_records_version", this.getGrantRecordsVersion()); + map.put("location_without_scheme", this.getLocationWithoutScheme()); return map; } @@ -293,29 +305,52 @@ public Builder grantRecordsVersion(int grantRecordsVersion) { return this; } + public Builder locationWithoutScheme(String location) { + entity.locationWithoutScheme = location; + return this; + } + public ModelEntity build() { return entity; } } public static ModelEntity fromEntity(PolarisBaseEntity entity) { - return ModelEntity.builder() - .catalogId(entity.getCatalogId()) - .id(entity.getId()) - .parentId(entity.getParentId()) - .typeCode(entity.getTypeCode()) - .name(entity.getName()) - .entityVersion(entity.getEntityVersion()) - .subTypeCode(entity.getSubTypeCode()) - .createTimestamp(entity.getCreateTimestamp()) - .dropTimestamp(entity.getDropTimestamp()) - .purgeTimestamp(entity.getPurgeTimestamp()) - .toPurgeTimestamp(entity.getToPurgeTimestamp()) - .lastUpdateTimestamp(entity.getLastUpdateTimestamp()) - .properties(entity.getProperties()) - .internalProperties(entity.getInternalProperties()) - .grantRecordsVersion(entity.getGrantRecordsVersion()) - .build(); + var builder = + ModelEntity.builder() + .catalogId(entity.getCatalogId()) + .id(entity.getId()) + .parentId(entity.getParentId()) + .typeCode(entity.getTypeCode()) + .name(entity.getName()) + .entityVersion(entity.getEntityVersion()) + .subTypeCode(entity.getSubTypeCode()) + .createTimestamp(entity.getCreateTimestamp()) + .dropTimestamp(entity.getDropTimestamp()) + .purgeTimestamp(entity.getPurgeTimestamp()) + .toPurgeTimestamp(entity.getToPurgeTimestamp()) + .lastUpdateTimestamp(entity.getLastUpdateTimestamp()) + .properties(entity.getProperties()) + .internalProperties(entity.getInternalProperties()) + .grantRecordsVersion(entity.getGrantRecordsVersion()); + + if (entity.getType() == PolarisEntityType.TABLE_LIKE) { + if (entity.getSubType() == PolarisEntitySubType.ICEBERG_TABLE + || entity.getSubType() == PolarisEntitySubType.ICEBERG_VIEW) { + builder.locationWithoutScheme( + StorageLocation.of( + entity.getPropertiesAsMap().get(PolarisEntityConstants.ENTITY_BASE_LOCATION)) + .withoutScheme()); + } + } + if (entity.getType() == PolarisEntityType.NAMESPACE) { + builder.locationWithoutScheme( + StorageLocation.of( + entity.getPropertiesAsMap().get(PolarisEntityConstants.ENTITY_BASE_LOCATION)) + .withoutScheme()); + } + + return builder.build(); } public static PolarisBaseEntity toEntity(ModelEntity model) { diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/SchemaVersion.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/SchemaVersion.java new file mode 100644 index 0000000000..7b885e12bd --- /dev/null +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/SchemaVersion.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.persistence.relational.jdbc.models; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Map; +import org.apache.polaris.persistence.relational.jdbc.DatabaseType; + +public class SchemaVersion implements Converter { + private final Integer value; + + public SchemaVersion() { + this.value = null; + } + + private SchemaVersion(int value) { + this.value = value; + } + + public int getValue() { + if (value == null) { + throw new IllegalStateException("Schema version should be constructed via fromResultSet"); + } + return value; + } + + @Override + public SchemaVersion fromResultSet(ResultSet rs) throws SQLException { + return new SchemaVersion(rs.getInt("version_value")); + } + + @Override + public Map toMap(DatabaseType databaseType) { + return Map.of("version_value", this.value); + } +} diff --git a/persistence/relational-jdbc/src/main/resources/h2/schema-v1.sql b/persistence/relational-jdbc/src/main/resources/h2/schema-v1.sql index cda0352efb..10b4e3774b 100644 --- a/persistence/relational-jdbc/src/main/resources/h2/schema-v1.sql +++ b/persistence/relational-jdbc/src/main/resources/h2/schema-v1.sql @@ -19,6 +19,19 @@ CREATE SCHEMA IF NOT EXISTS POLARIS_SCHEMA; SET SCHEMA POLARIS_SCHEMA; + +CREATE TABLE IF NOT EXISTS version ( + version_key VARCHAR PRIMARY KEY, + version_value INTEGER NOT NULL +); + +MERGE INTO version (version_key, version_value) + KEY (version_key) + VALUES ('version', 1); + +-- H2 supports COMMENT, but some modes may ignore it +COMMENT ON TABLE version IS 'the version of the JDBC schema in use'; + DROP TABLE IF EXISTS entities; CREATE TABLE IF NOT EXISTS entities ( realm_id TEXT NOT NULL, diff --git a/persistence/relational-jdbc/src/main/resources/h2/schema-v2.sql b/persistence/relational-jdbc/src/main/resources/h2/schema-v2.sql new file mode 100644 index 0000000000..6cbd6c03ae --- /dev/null +++ b/persistence/relational-jdbc/src/main/resources/h2/schema-v2.sql @@ -0,0 +1,127 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file-- +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"). You may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- + +-- Changes from v1: +-- * Added a `location_without_scheme` column to entities +-- * Added an index `idx_locations` over (realm_id, catalog_id, location_without_scheme) in entities + +CREATE SCHEMA IF NOT EXISTS POLARIS_SCHEMA; +SET SCHEMA POLARIS_SCHEMA; + +CREATE TABLE IF NOT EXISTS version ( + version_key VARCHAR PRIMARY KEY, + version_value INTEGER NOT NULL +); + +MERGE INTO version (version_key, version_value) + KEY (version_key) + VALUES ('version', 2); + +-- H2 supports COMMENT, but some modes may ignore it +COMMENT ON TABLE version IS 'the version of the JDBC schema in use'; + +DROP TABLE IF EXISTS entities; +CREATE TABLE IF NOT EXISTS entities ( + realm_id TEXT NOT NULL, + catalog_id BIGINT NOT NULL, + id BIGINT NOT NULL, + parent_id BIGINT NOT NULL, + name TEXT NOT NULL, + entity_version INT NOT NULL, + type_code INT NOT NULL, + sub_type_code INT NOT NULL, + create_timestamp BIGINT NOT NULL, + drop_timestamp BIGINT NOT NULL, + purge_timestamp BIGINT NOT NULL, + to_purge_timestamp BIGINT NOT NULL, + last_update_timestamp BIGINT NOT NULL, + properties TEXT NOT NULL DEFAULT '{}', + internal_properties TEXT NOT NULL DEFAULT '{}', + grant_records_version INT NOT NULL, + location_without_scheme TEXT, + PRIMARY KEY (realm_id, id), + CONSTRAINT constraint_name UNIQUE (realm_id, catalog_id, parent_id, type_code, name) +); + +CREATE INDEX IF NOT EXISTS idx_locations ON entities(realm_id, catalog_id, location_without_scheme); + +-- TODO: create indexes based on all query pattern. +CREATE INDEX IF NOT EXISTS idx_entities ON entities (realm_id, catalog_id, id); + +COMMENT ON TABLE entities IS 'all the entities'; + +COMMENT ON COLUMN entities.catalog_id IS 'catalog id'; +COMMENT ON COLUMN entities.id IS 'entity id'; +COMMENT ON COLUMN entities.parent_id IS 'entity id of parent'; +COMMENT ON COLUMN entities.name IS 'entity name'; +COMMENT ON COLUMN entities.entity_version IS 'version of the entity'; +COMMENT ON COLUMN entities.type_code IS 'type code'; +COMMENT ON COLUMN entities.sub_type_code IS 'sub type of entity'; +COMMENT ON COLUMN entities.create_timestamp IS 'creation time of entity'; +COMMENT ON COLUMN entities.drop_timestamp IS 'time of drop of entity'; +COMMENT ON COLUMN entities.purge_timestamp IS 'time to start purging entity'; +COMMENT ON COLUMN entities.last_update_timestamp IS 'last time the entity is touched'; +COMMENT ON COLUMN entities.properties IS 'entities properties json'; +COMMENT ON COLUMN entities.internal_properties IS 'entities internal properties json'; +COMMENT ON COLUMN entities.grant_records_version IS 'the version of grant records change on the entity'; + +DROP TABLE IF EXISTS grant_records; +CREATE TABLE IF NOT EXISTS grant_records ( + realm_id TEXT NOT NULL, + securable_catalog_id BIGINT NOT NULL, + securable_id BIGINT NOT NULL, + grantee_catalog_id BIGINT NOT NULL, + grantee_id BIGINT NOT NULL, + privilege_code INTEGER, + PRIMARY KEY (realm_id, securable_catalog_id, securable_id, grantee_catalog_id, grantee_id, privilege_code) +); + +COMMENT ON TABLE grant_records IS 'grant records for entities'; +COMMENT ON COLUMN grant_records.securable_catalog_id IS 'catalog id of the securable'; +COMMENT ON COLUMN grant_records.securable_id IS 'entity id of the securable'; +COMMENT ON COLUMN grant_records.grantee_catalog_id IS 'catalog id of the grantee'; +COMMENT ON COLUMN grant_records.grantee_id IS 'id of the grantee'; +COMMENT ON COLUMN grant_records.privilege_code IS 'privilege code'; + +DROP TABLE IF EXISTS principal_authentication_data; +CREATE TABLE IF NOT EXISTS principal_authentication_data ( + realm_id TEXT NOT NULL, + principal_id BIGINT NOT NULL, + principal_client_id VARCHAR(255) NOT NULL, + main_secret_hash VARCHAR(255) NOT NULL, + secondary_secret_hash VARCHAR(255) NOT NULL, + secret_salt VARCHAR(255) NOT NULL, + PRIMARY KEY (realm_id, principal_client_id) +); + +COMMENT ON TABLE principal_authentication_data IS 'authentication data for client'; + +DROP TABLE IF EXISTS policy_mapping_record; +CREATE TABLE IF NOT EXISTS policy_mapping_record ( + realm_id TEXT NOT NULL, + target_catalog_id BIGINT NOT NULL, + target_id BIGINT NOT NULL, + policy_type_code INTEGER NOT NULL, + policy_catalog_id BIGINT NOT NULL, + policy_id BIGINT NOT NULL, + parameters TEXT NOT NULL DEFAULT '{}', + PRIMARY KEY (realm_id, target_catalog_id, target_id, policy_type_code, policy_catalog_id, policy_id) +); + +CREATE INDEX IF NOT EXISTS idx_policy_mapping_record ON policy_mapping_record (realm_id, policy_type_code, policy_catalog_id, policy_id, target_catalog_id, target_id); diff --git a/persistence/relational-jdbc/src/main/resources/postgres/schema-v1.sql b/persistence/relational-jdbc/src/main/resources/postgres/schema-v1.sql index c9fcbc4488..0b02312baa 100644 --- a/persistence/relational-jdbc/src/main/resources/postgres/schema-v1.sql +++ b/persistence/relational-jdbc/src/main/resources/postgres/schema-v1.sql @@ -19,6 +19,16 @@ CREATE SCHEMA IF NOT EXISTS POLARIS_SCHEMA; SET search_path TO POLARIS_SCHEMA; +CREATE TABLE IF NOT EXISTS version ( + version_key TEXT PRIMARY KEY, + version_value INTEGER NOT NULL +); +INSERT INTO version (version_key, version_value) +VALUES ('version', 1) + ON CONFLICT (version_key) DO UPDATE + SET version_value = EXCLUDED.version_value; +COMMENT ON TABLE version IS 'the version of the JDBC schema in use'; + CREATE TABLE IF NOT EXISTS entities ( realm_id TEXT NOT NULL, catalog_id BIGINT NOT NULL, diff --git a/persistence/relational-jdbc/src/main/resources/postgres/schema-v2.sql b/persistence/relational-jdbc/src/main/resources/postgres/schema-v2.sql new file mode 100644 index 0000000000..638708d83b --- /dev/null +++ b/persistence/relational-jdbc/src/main/resources/postgres/schema-v2.sql @@ -0,0 +1,124 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file-- +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"). You may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Changes from v1: +-- * Added a `location` column to entities +-- * Added an index `idx_locations` over (realm_id, parent_id, location) in entities + +CREATE SCHEMA IF NOT EXISTS POLARIS_SCHEMA; +SET search_path TO POLARIS_SCHEMA; + +CREATE TABLE IF NOT EXISTS version ( + version_key TEXT PRIMARY KEY, + version_value INTEGER NOT NULL +); +INSERT INTO version (version_key, version_value) +VALUES ('version', 2) +ON CONFLICT (version_key) DO UPDATE +SET version_value = EXCLUDED.version_value; +COMMENT ON TABLE version IS 'the version of the JDBC schema in use'; + +CREATE TABLE IF NOT EXISTS entities ( + realm_id TEXT NOT NULL, + catalog_id BIGINT NOT NULL, + id BIGINT NOT NULL, + parent_id BIGINT NOT NULL, + name TEXT NOT NULL, + entity_version INT NOT NULL, + type_code INT NOT NULL, + sub_type_code INT NOT NULL, + create_timestamp BIGINT NOT NULL, + drop_timestamp BIGINT NOT NULL, + purge_timestamp BIGINT NOT NULL, + to_purge_timestamp BIGINT NOT NULL, + last_update_timestamp BIGINT NOT NULL, + properties JSONB not null default '{}'::JSONB, + internal_properties JSONB not null default '{}'::JSONB, + grant_records_version INT NOT NULL, + location_without_scheme TEXT, + PRIMARY KEY (realm_id, id), + CONSTRAINT constraint_name UNIQUE (realm_id, catalog_id, parent_id, type_code, name) +); + +-- TODO: create indexes based on all query pattern. +CREATE INDEX IF NOT EXISTS idx_entities ON entities (realm_id, catalog_id, id); +CREATE INDEX IF NOT EXISTS idx_locations + ON entities USING btree (realm_id, parent_id, location_without_scheme) + WHERE location_without_scheme IS NOT NULL; + +COMMENT ON TABLE entities IS 'all the entities'; + +COMMENT ON COLUMN entities.realm_id IS 'realm_id used for multi-tenancy'; +COMMENT ON COLUMN entities.catalog_id IS 'catalog id'; +COMMENT ON COLUMN entities.id IS 'entity id'; +COMMENT ON COLUMN entities.parent_id IS 'entity id of parent'; +COMMENT ON COLUMN entities.name IS 'entity name'; +COMMENT ON COLUMN entities.entity_version IS 'version of the entity'; +COMMENT ON COLUMN entities.type_code IS 'type code'; +COMMENT ON COLUMN entities.sub_type_code IS 'sub type of entity'; +COMMENT ON COLUMN entities.create_timestamp IS 'creation time of entity'; +COMMENT ON COLUMN entities.drop_timestamp IS 'time of drop of entity'; +COMMENT ON COLUMN entities.purge_timestamp IS 'time to start purging entity'; +COMMENT ON COLUMN entities.last_update_timestamp IS 'last time the entity is touched'; +COMMENT ON COLUMN entities.properties IS 'entities properties json'; +COMMENT ON COLUMN entities.internal_properties IS 'entities internal properties json'; +COMMENT ON COLUMN entities.grant_records_version IS 'the version of grant records change on the entity'; + +CREATE TABLE IF NOT EXISTS grant_records ( + realm_id TEXT NOT NULL, + securable_catalog_id BIGINT NOT NULL, + securable_id BIGINT NOT NULL, + grantee_catalog_id BIGINT NOT NULL, + grantee_id BIGINT NOT NULL, + privilege_code INTEGER, + PRIMARY KEY (realm_id, securable_catalog_id, securable_id, grantee_catalog_id, grantee_id, privilege_code) +); + +COMMENT ON TABLE grant_records IS 'grant records for entities'; + +COMMENT ON COLUMN grant_records.securable_catalog_id IS 'catalog id of the securable'; +COMMENT ON COLUMN grant_records.securable_id IS 'entity id of the securable'; +COMMENT ON COLUMN grant_records.grantee_catalog_id IS 'catalog id of the grantee'; +COMMENT ON COLUMN grant_records.grantee_id IS 'id of the grantee'; +COMMENT ON COLUMN grant_records.privilege_code IS 'privilege code'; + +CREATE TABLE IF NOT EXISTS principal_authentication_data ( + realm_id TEXT NOT NULL, + principal_id BIGINT NOT NULL, + principal_client_id VARCHAR(255) NOT NULL, + main_secret_hash VARCHAR(255) NOT NULL, + secondary_secret_hash VARCHAR(255) NOT NULL, + secret_salt VARCHAR(255) NOT NULL, + PRIMARY KEY (realm_id, principal_client_id) +); + +COMMENT ON TABLE principal_authentication_data IS 'authentication data for client'; + +DROP TABLE IF EXISTS policy_mapping_record; +CREATE TABLE IF NOT EXISTS policy_mapping_record ( + realm_id TEXT NOT NULL, + target_catalog_id BIGINT NOT NULL, + target_id BIGINT NOT NULL, + policy_type_code INTEGER NOT NULL, + policy_catalog_id BIGINT NOT NULL, + policy_id BIGINT NOT NULL, + parameters JSONB NOT NULL DEFAULT '{}'::JSONB, + PRIMARY KEY (realm_id, target_catalog_id, target_id, policy_type_code, policy_catalog_id, policy_id) +); + +CREATE INDEX IF NOT EXISTS idx_policy_mapping_record ON policy_mapping_record (realm_id, policy_type_code, policy_catalog_id, policy_id, target_catalog_id, target_id); diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java index fbcd3a958b..a824a17a56 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java @@ -50,7 +50,7 @@ protected PolarisTestMetaStoreManager createPolarisTestMetaStoreManager() { datasourceOperations = new DatasourceOperations(createH2DataSource(), new H2JdbcConfiguration()); datasourceOperations.executeScript( - String.format("%s/schema-v1.sql", DatabaseType.H2.getDisplayName())); + String.format("%s/schema-v2.sql", DatabaseType.H2.getDisplayName())); } catch (SQLException e) { throw new RuntimeException( String.format( diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/QueryGeneratorTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/QueryGeneratorTest.java index c7a589ca5d..798dd92e7d 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/QueryGeneratorTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/QueryGeneratorTest.java @@ -32,6 +32,7 @@ import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; import org.apache.polaris.persistence.relational.jdbc.models.ModelEntity; +import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; public class QueryGeneratorTest { @@ -44,7 +45,7 @@ void testGenerateSelectQuery_withMaQueryGeneratorpWhereClause() { whereClause.put("name", "testEntity"); whereClause.put("entity_version", 1); String expectedQuery = - "SELECT id, catalog_id, parent_id, type_code, name, entity_version, sub_type_code, create_timestamp, drop_timestamp, purge_timestamp, to_purge_timestamp, last_update_timestamp, properties, internal_properties, grant_records_version FROM POLARIS_SCHEMA.ENTITIES WHERE entity_version = ? AND name = ?"; + "SELECT id, catalog_id, parent_id, type_code, name, entity_version, sub_type_code, create_timestamp, drop_timestamp, purge_timestamp, to_purge_timestamp, last_update_timestamp, properties, internal_properties, grant_records_version, location_without_scheme FROM POLARIS_SCHEMA.ENTITIES WHERE entity_version = ? AND name = ?"; assertEquals( expectedQuery, QueryGenerator.generateSelectQuery( @@ -71,7 +72,7 @@ void testGenerateDeleteQueryForEntityGrantRecords() { void testGenerateSelectQueryWithEntityIds_singleId() { List entityIds = Collections.singletonList(new PolarisEntityId(123L, 1L)); String expectedQuery = - "SELECT id, catalog_id, parent_id, type_code, name, entity_version, sub_type_code, create_timestamp, drop_timestamp, purge_timestamp, to_purge_timestamp, last_update_timestamp, properties, internal_properties, grant_records_version FROM POLARIS_SCHEMA.ENTITIES WHERE (catalog_id, id) IN ((?, ?)) AND realm_id = ?"; + "SELECT id, catalog_id, parent_id, type_code, name, entity_version, sub_type_code, create_timestamp, drop_timestamp, purge_timestamp, to_purge_timestamp, last_update_timestamp, properties, internal_properties, grant_records_version, location_without_scheme FROM POLARIS_SCHEMA.ENTITIES WHERE (catalog_id, id) IN ((?, ?)) AND realm_id = ?"; assertEquals( expectedQuery, QueryGenerator.generateSelectQueryWithEntityIds(REALM_ID, entityIds).sql()); } @@ -81,7 +82,7 @@ void testGenerateSelectQueryWithEntityIds_multipleIds() { List entityIds = Arrays.asList(new PolarisEntityId(123L, 1L), new PolarisEntityId(456L, 2L)); String expectedQuery = - "SELECT id, catalog_id, parent_id, type_code, name, entity_version, sub_type_code, create_timestamp, drop_timestamp, purge_timestamp, to_purge_timestamp, last_update_timestamp, properties, internal_properties, grant_records_version FROM POLARIS_SCHEMA.ENTITIES WHERE (catalog_id, id) IN ((?, ?), (?, ?)) AND realm_id = ?"; + "SELECT id, catalog_id, parent_id, type_code, name, entity_version, sub_type_code, create_timestamp, drop_timestamp, purge_timestamp, to_purge_timestamp, last_update_timestamp, properties, internal_properties, grant_records_version, location_without_scheme FROM POLARIS_SCHEMA.ENTITIES WHERE (catalog_id, id) IN ((?, ?), (?, ?)) AND realm_id = ?"; assertEquals( expectedQuery, QueryGenerator.generateSelectQueryWithEntityIds(REALM_ID, entityIds).sql()); } @@ -98,22 +99,7 @@ void testGenerateSelectQueryWithEntityIds_emptyList() { void testGenerateInsertQuery_nonNullFields() { ModelEntity entity = ModelEntity.builder().name("test").entityVersion(1).build(); String expectedQuery = - "INSERT INTO POLARIS_SCHEMA.ENTITIES (id, catalog_id, parent_id, type_code, name, entity_version, sub_type_code, create_timestamp, drop_timestamp, purge_timestamp, to_purge_timestamp, last_update_timestamp, properties, internal_properties, grant_records_version, realm_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; - assertEquals( - expectedQuery, - QueryGenerator.generateInsertQuery( - ModelEntity.ALL_COLUMNS, - ModelEntity.TABLE_NAME, - entity.toMap(DatabaseType.H2).values().stream().toList(), - REALM_ID) - .sql()); - } - - @Test - void testGenerateInsertQuery_nullFields() { - ModelEntity entity = ModelEntity.builder().name("test").build(); - String expectedQuery = - "INSERT INTO POLARIS_SCHEMA.ENTITIES (id, catalog_id, parent_id, type_code, name, entity_version, sub_type_code, create_timestamp, drop_timestamp, purge_timestamp, to_purge_timestamp, last_update_timestamp, properties, internal_properties, grant_records_version, realm_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + "INSERT INTO POLARIS_SCHEMA.ENTITIES (id, catalog_id, parent_id, type_code, name, entity_version, sub_type_code, create_timestamp, drop_timestamp, purge_timestamp, to_purge_timestamp, last_update_timestamp, properties, internal_properties, grant_records_version, location_without_scheme, realm_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; assertEquals( expectedQuery, QueryGenerator.generateInsertQuery( @@ -130,7 +116,7 @@ void testGenerateUpdateQuery_nonNullFields() { Map whereClause = new HashMap<>(); whereClause.put("id", 123L); String expectedQuery = - "UPDATE POLARIS_SCHEMA.ENTITIES SET id = ?, catalog_id = ?, parent_id = ?, type_code = ?, name = ?, entity_version = ?, sub_type_code = ?, create_timestamp = ?, drop_timestamp = ?, purge_timestamp = ?, to_purge_timestamp = ?, last_update_timestamp = ?, properties = ?, internal_properties = ?, grant_records_version = ? WHERE id = ?"; + "UPDATE POLARIS_SCHEMA.ENTITIES SET id = ?, catalog_id = ?, parent_id = ?, type_code = ?, name = ?, entity_version = ?, sub_type_code = ?, create_timestamp = ?, drop_timestamp = ?, purge_timestamp = ?, to_purge_timestamp = ?, last_update_timestamp = ?, properties = ?, internal_properties = ?, grant_records_version = ?, location_without_scheme = ? WHERE id = ?"; assertEquals( expectedQuery, QueryGenerator.generateUpdateQuery( @@ -147,7 +133,7 @@ void testGenerateUpdateQuery_partialNonNullFields() { Map whereClause = new HashMap<>(); whereClause.put("id", 123L); String expectedQuery = - "UPDATE POLARIS_SCHEMA.ENTITIES SET id = ?, catalog_id = ?, parent_id = ?, type_code = ?, name = ?, entity_version = ?, sub_type_code = ?, create_timestamp = ?, drop_timestamp = ?, purge_timestamp = ?, to_purge_timestamp = ?, last_update_timestamp = ?, properties = ?, internal_properties = ?, grant_records_version = ? WHERE id = ?"; + "UPDATE POLARIS_SCHEMA.ENTITIES SET id = ?, catalog_id = ?, parent_id = ?, type_code = ?, name = ?, entity_version = ?, sub_type_code = ?, create_timestamp = ?, drop_timestamp = ?, purge_timestamp = ?, to_purge_timestamp = ?, last_update_timestamp = ?, properties = ?, internal_properties = ?, grant_records_version = ?, location_without_scheme = ? WHERE id = ?"; assertEquals( expectedQuery, QueryGenerator.generateUpdateQuery( @@ -186,20 +172,7 @@ void testGenerateDeleteQuery_byObject() { Map objMap = entityToDelete.toMap(DatabaseType.H2); objMap.put("realm_id", REALM_ID); String expectedQuery = - "DELETE FROM POLARIS_SCHEMA.ENTITIES WHERE id = ? AND catalog_id = ? AND parent_id = ? AND type_code = ? AND name = ? AND entity_version = ? AND sub_type_code = ? AND create_timestamp = ? AND drop_timestamp = ? AND purge_timestamp = ? AND to_purge_timestamp = ? AND last_update_timestamp = ? AND properties = ? AND internal_properties = ? AND grant_records_version = ? AND realm_id = ?"; - assertEquals( - expectedQuery, - QueryGenerator.generateDeleteQuery(ModelEntity.ALL_COLUMNS, ModelEntity.TABLE_NAME, objMap) - .sql()); - } - - @Test - void testGenerateDeleteQuery_byObject_nullValue() { - ModelEntity entityToDelete = ModelEntity.builder().name("test").dropTimestamp(0L).build(); - Map objMap = entityToDelete.toMap(DatabaseType.H2); - objMap.put("realm_id", REALM_ID); - String expectedQuery = - "DELETE FROM POLARIS_SCHEMA.ENTITIES WHERE id = ? AND catalog_id = ? AND parent_id = ? AND type_code = ? AND name = ? AND entity_version = ? AND sub_type_code = ? AND create_timestamp = ? AND drop_timestamp = ? AND purge_timestamp = ? AND to_purge_timestamp = ? AND last_update_timestamp = ? AND properties = ? AND internal_properties = ? AND grant_records_version = ? AND realm_id = ?"; + "DELETE FROM POLARIS_SCHEMA.ENTITIES WHERE id = ? AND catalog_id = ? AND parent_id = ? AND type_code = ? AND name = ? AND entity_version = ? AND sub_type_code = ? AND create_timestamp = ? AND drop_timestamp = ? AND purge_timestamp = ? AND to_purge_timestamp = ? AND last_update_timestamp = ? AND properties = ? AND internal_properties = ? AND grant_records_version = ? AND location_without_scheme = ? AND realm_id = ?"; assertEquals( expectedQuery, QueryGenerator.generateDeleteQuery(ModelEntity.ALL_COLUMNS, ModelEntity.TABLE_NAME, objMap) @@ -229,4 +202,53 @@ void testGenerateWhereClause_emptyMap() { Map whereClause = Collections.emptyMap(); assertEquals("", QueryGenerator.generateWhereClause(Set.of(), whereClause).sql()); } + + @Test + void testGenerateOverlapQuery() { + assertEquals( + "SELECT id, catalog_id, parent_id, type_code, name, entity_version, sub_type_code," + + " create_timestamp, drop_timestamp, purge_timestamp, to_purge_timestamp, last_update_timestamp," + + " properties, internal_properties, grant_records_version, location_without_scheme FROM" + + " POLARIS_SCHEMA.ENTITIES WHERE realm_id = ? AND catalog_id = ? AND (location_without_scheme = ?" + + " OR location_without_scheme = ? OR location_without_scheme = ? OR location_without_scheme = ? OR" + + " location_without_scheme = ? OR location_without_scheme LIKE ?)", + QueryGenerator.generateOverlapQuery("realmId", -123, "s3://bucket/tmp/location/").sql()); + Assertions.assertThatCollection( + QueryGenerator.generateOverlapQuery("realmId", -123, "s3://bucket/tmp/location/") + .parameters()) + .containsExactly( + "realmId", + -123L, + "/", + "//", + "//bucket/", + "//bucket/tmp/", + "//bucket/tmp/location/", + "//bucket/tmp/location/%"); + + assertEquals( + "SELECT id, catalog_id, parent_id, type_code, name, entity_version, sub_type_code," + + " create_timestamp, drop_timestamp, purge_timestamp, to_purge_timestamp, last_update_timestamp," + + " properties, internal_properties, grant_records_version, location_without_scheme FROM" + + " POLARIS_SCHEMA.ENTITIES WHERE realm_id = ? AND catalog_id = ? AND (location_without_scheme = ? OR location_without_scheme = ?" + + " OR location_without_scheme = ? OR location_without_scheme = ? OR location_without_scheme = ? OR location_without_scheme LIKE ?)", + QueryGenerator.generateOverlapQuery("realmId", -123, "/tmp/location/").sql()); + Assertions.assertThatCollection( + QueryGenerator.generateOverlapQuery("realmId", -123, "/tmp/location/").parameters()) + .containsExactly( + "realmId", -123L, "/", "//", "///", "///tmp/", "///tmp/location/", "///tmp/location/%"); + + assertEquals( + "SELECT id, catalog_id, parent_id, type_code, name, entity_version, sub_type_code," + + " create_timestamp, drop_timestamp, purge_timestamp, to_purge_timestamp, last_update_timestamp," + + " properties, internal_properties, grant_records_version, location_without_scheme" + + " FROM POLARIS_SCHEMA.ENTITIES WHERE realm_id = ? AND catalog_id = ? AND (location_without_scheme = ?" + + " OR location_without_scheme = ? OR location_without_scheme = ? OR location_without_scheme = ? OR location_without_scheme LIKE ?)", + QueryGenerator.generateOverlapQuery("realmId", -123, "s3://バケツ/\"loc.ation\"/").sql()); + Assertions.assertThatCollection( + QueryGenerator.generateOverlapQuery("realmId", -123, "s3://バケツ/\"loc.ation\"/") + .parameters()) + .containsExactly( + "realmId", -123L, "/", "//", "//バケツ/", "//バケツ/\"loc.ation\"/", "//バケツ/\"loc.ation\"/%"); + } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java index fe265c3072..b9f82b624b 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java @@ -283,7 +283,7 @@ public static void enforceFeatureEnabledOrThrow( .key("ALLOW_INSECURE_STORAGE_TYPES") .description( "Allow usage of FileIO implementations that are considered insecure. " - + "Enabling this setting may expose the service to possibly severe security risks!" + + "Enabling this setting may expose the service to possibly severe security risks! " + "This should only be set to 'true' for tests!") .defaultValue(false) .buildFeatureConfiguration(); @@ -298,4 +298,25 @@ public static void enforceFeatureEnabledOrThrow( + "in their snapshot summary") .defaultValue(false) .buildFeatureConfiguration(); + + public static final FeatureConfiguration ADD_TRAILING_SLASH_TO_LOCATION = + PolarisConfiguration.builder() + .key("ADD_TRAILING_SLASH_TO_LOCATION") + .catalogConfig("polaris.config.add-trailing-slash-to-location") + .description( + "When set, the base location for a table or namespace will have `/` added as a suffix if not present") + .defaultValue(true) + .buildFeatureConfiguration(); + + public static final FeatureConfiguration OPTIMIZED_SIBLING_CHECK = + PolarisConfiguration.builder() + .key("OPTIMIZED_SIBLING_CHECK") + .description( + "When set, an index is used to perform the sibling check between tables, views, and namespaces. New " + + "locations will be checked against previous ones based on components, so the new location " + + "/foo/bar/ will check for a sibling at /, /foo/ and /foo/bar/%. In order for this check to " + + "be correct, locations should end with a slash. See ADD_TRAILING_SLASH_TO_LOCATION for a way " + + "to enforce this when new locations are added. Only supported by the JDBC metastore.") + .defaultValue(false) + .buildFeatureConfiguration(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionType.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionType.java index 56796a95c4..441c0c4c53 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionType.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionType.java @@ -26,7 +26,7 @@ * the API model. We define integer type codes in this enum for better compatibility within * persisted data in case the names of enum types are ever changed in place. * - *

Important: Codes must be kept in-sync with JsonSubTypes annotated within {@link + *

Important: Codes must be kept in-sync wi th JsonSubTypes annotated within {@link * ConnectionConfigInfoDpo}. */ public enum ConnectionType { diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java index f975ae7392..99d8557f0b 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java @@ -55,7 +55,7 @@ * Catalog specific subclass of the {@link PolarisEntity} that handles conversion from the {@link * Catalog} model to the persistent entity model. */ -public class CatalogEntity extends PolarisEntity { +public class CatalogEntity extends PolarisEntity implements LocationBasedEntity { public static final String CATALOG_TYPE_PROPERTY = "catalogType"; // Specifies the object-store base location used for all Table file locations under the @@ -91,7 +91,7 @@ public static CatalogEntity fromCatalog(CallContext callContext, Catalog catalog internalProperties.put(CATALOG_TYPE_PROPERTY, catalog.getType().name()); builder.setInternalProperties(internalProperties); builder.setStorageConfigurationInfo( - callContext, catalog.getStorageConfigInfo(), getDefaultBaseLocation(catalog)); + callContext, catalog.getStorageConfigInfo(), getBaseLocation(catalog)); return builder.build(); } @@ -179,7 +179,8 @@ private ConnectionConfigInfo getConnectionInfo(Map internalPrope return null; } - public String getDefaultBaseLocation() { + @Override + public String getBaseLocation() { return getPropertiesAsMap().get(DEFAULT_BASE_LOCATION_KEY); } @@ -342,7 +343,7 @@ public CatalogEntity build() { } } - protected static @Nonnull String getDefaultBaseLocation(Catalog catalog) { + protected static @Nonnull String getBaseLocation(Catalog catalog) { return catalog.getProperties().getDefaultBaseLocation(); } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/LocationBasedEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/LocationBasedEntity.java new file mode 100644 index 0000000000..93d4384724 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/LocationBasedEntity.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.entity; + +/** + * An interface for entity types that correspond to some storage location. These entities provide a + * method `getBaseLocation` to retrieve that location. + */ +public interface LocationBasedEntity { + + /** + * @return The base location for this entity + */ + String getBaseLocation(); +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/NamespaceEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/NamespaceEntity.java index 7fa65042a9..b1fb3dae0a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/NamespaceEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/NamespaceEntity.java @@ -27,7 +27,7 @@ * Namespace-specific subclass of the {@link PolarisEntity} that provides accessors interacting with * internalProperties specific to the NAMESPACE type. */ -public class NamespaceEntity extends PolarisEntity { +public class NamespaceEntity extends PolarisEntity implements LocationBasedEntity { // RESTUtil-encoded parent namespace. public static final String PARENT_NAMESPACE_KEY = "parent-namespace"; @@ -60,6 +60,7 @@ public Namespace asNamespace() { return Namespace.of(levels); } + @Override @JsonIgnore public String getBaseLocation() { return getPropertiesAsMap().get(PolarisEntityConstants.ENTITY_BASE_LOCATION); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/table/IcebergTableLikeEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/table/IcebergTableLikeEntity.java index 026a569b34..21e9229829 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/table/IcebergTableLikeEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/table/IcebergTableLikeEntity.java @@ -23,6 +23,7 @@ import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.rest.RESTUtil; +import org.apache.polaris.core.entity.LocationBasedEntity; import org.apache.polaris.core.entity.NamespaceEntity; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntity; @@ -33,7 +34,7 @@ * An entity type for {@link TableLikeEntity} instances that conform to iceberg semantics around * locations. This includes both Iceberg tables and Iceberg views. */ -public class IcebergTableLikeEntity extends TableLikeEntity { +public class IcebergTableLikeEntity extends TableLikeEntity implements LocationBasedEntity { // For applicable types, this key on the "internalProperties" map will return the location // of the internalProperties JSON file. public static final String METADATA_LOCATION_KEY = "metadata-location"; @@ -67,6 +68,7 @@ public Optional getLastAdmittedNotificationTimestamp() { .map(Long::parseLong); } + @Override @JsonIgnore public String getBaseLocation() { return getPropertiesAsMap().get(PolarisEntityConstants.ENTITY_BASE_LOCATION); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java index 4f5a98ce3c..0c9f3cc10d 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; @@ -34,6 +35,7 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.entity.AsyncTaskType; import org.apache.polaris.core.entity.EntityNameLookupRecord; +import org.apache.polaris.core.entity.LocationBasedEntity; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntity; @@ -1818,6 +1820,15 @@ public Map getInternalPropertyMap( return new ResolvedEntityResult(entity, entityVersions.getGrantRecordsVersion(), grantRecords); } + /** {@inheritDoc} */ + @Override + public + Optional> hasOverlappingSiblings( + @Nonnull PolarisCallContext callContext, T entity) { + BasePersistence ms = callContext.getMetaStore(); + return ms.hasOverlappingSiblings(callContext, entity); + } + @Override public @Nonnull PolicyAttachmentResult attachPolicyToEntity( @Nonnull PolarisCallContext callCtx, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java index dc85f2183b..26ccd8de39 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java @@ -21,12 +21,15 @@ import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; import java.util.List; +import java.util.Optional; import java.util.function.Function; import java.util.function.Predicate; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.entity.EntityNameLookupRecord; +import org.apache.polaris.core.entity.LocationBasedEntity; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; +import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; import org.apache.polaris.core.entity.PolarisEntityType; @@ -403,6 +406,22 @@ boolean hasChildren( long catalogId, long parentId); + /** + * Check if the specified IcebergTableLikeEntity / NamespaceEntity has any sibling entities which + * share a base location + * + * @param callContext the polaris call context + * @param entity the entity to check for overlapping siblings for + * @return Optional.of(Optional.of(location)) if the parent entity has children, + * Optional.of(Optional.empty()) if not, and Optional.empty() if the metastore doesn't support + * this operation + */ + default + Optional> hasOverlappingSiblings( + @Nonnull PolarisCallContext callContext, T entity) { + return Optional.empty(); + } + /** * Performs operations necessary to isolate the state of {@code this} {@link BasePersistence} * instance from the state of the returned instance as far as multithreaded usage is concerned. If diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java index 2a20ad5c1e..b2fec2ddd8 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java @@ -22,9 +22,11 @@ import jakarta.annotation.Nullable; import java.util.List; import java.util.Map; +import java.util.Optional; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.auth.PolarisGrantManager; import org.apache.polaris.core.auth.PolarisSecretsManager; +import org.apache.polaris.core.entity.LocationBasedEntity; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntityCore; @@ -391,6 +393,22 @@ ResolvedEntityResult refreshResolvedEntity( long entityCatalogId, long entityId); + /** + * Check if the specified IcebergTableLikeEntity has any same-namespace siblings which share a + * location + * + * @param callContext the polaris call context + * @param entity the entity to check for overlapping siblings for + * @return Optional.of(Optional.of ( location)) if the parent entity has children, + * Optional.of(Optional.empty()) if not, and Optional.empty() if the metastore doesn't support + * this operation + */ + default + Optional> hasOverlappingSiblings( + @Nonnull PolarisCallContext callContext, T entity) { + return Optional.empty(); + } + /** * Indicates whether this metastore manager implementation requires entities to be reloaded via * {@link #loadEntitiesChangeTracking} in order to ensure the most recent versions are obtained. diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/TransactionWorkspaceMetaStoreManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/TransactionWorkspaceMetaStoreManager.java index b7ba47e83e..a7138c9fe2 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/TransactionWorkspaceMetaStoreManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/TransactionWorkspaceMetaStoreManager.java @@ -24,8 +24,10 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.LocationBasedEntity; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntityCore; @@ -385,6 +387,17 @@ public ResolvedEntityResult refreshResolvedEntity( return null; } + /** {@inheritDoc} */ + @Override + public + Optional> hasOverlappingSiblings( + @Nonnull PolarisCallContext callContext, T entity) { + callContext + .getDiagServices() + .fail("illegal_method_in_transaction_workspace", "hasOverlappingSiblings"); + return Optional.empty(); + } + @Override public @Nonnull PolicyAttachmentResult attachPolicyToEntity( @Nonnull PolarisCallContext callCtx, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java index a81566c99a..8888c7f3ea 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java @@ -35,6 +35,7 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.entity.AsyncTaskType; import org.apache.polaris.core.entity.EntityNameLookupRecord; +import org.apache.polaris.core.entity.LocationBasedEntity; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntity; @@ -2323,6 +2324,14 @@ public Map getInternalPropertyMap( entityId)); } + /** {@inheritDoc} */ + @Override + public + Optional> hasOverlappingSiblings( + @Nonnull PolarisCallContext callContext, T entity) { + return Optional.empty(); + } + /** {@inheritDoc} */ @Override public @Nonnull PolicyAttachmentResult attachPolicyToEntity( diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TreeMapTransactionalPersistenceImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TreeMapTransactionalPersistenceImpl.java index 44b37c2760..bf0517a1bb 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TreeMapTransactionalPersistenceImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TreeMapTransactionalPersistenceImpl.java @@ -22,6 +22,7 @@ import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; import java.util.List; +import java.util.Optional; import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; @@ -29,9 +30,11 @@ import java.util.stream.Stream; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.entity.EntityNameLookupRecord; +import org.apache.polaris.core.entity.LocationBasedEntity; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; +import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; import org.apache.polaris.core.entity.PolarisEntityType; @@ -663,4 +666,12 @@ record -> this.store.getSlicePolicyMappingRecordsByPolicy().delete(record)); .getSlicePolicyMappingRecordsByPolicy() .readRange(this.store.buildPrefixKeyComposite(policyTypeCode, policyCatalogId, policyId)); } + + /** {@inheritDoc} */ + @Override + public + Optional> hasOverlappingSiblings( + @Nonnull PolarisCallContext callContext, T entity) { + return Optional.empty(); + } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageLocation.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageLocation.java index c9f731a8cd..7bc768b6d8 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageLocation.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageLocation.java @@ -20,13 +20,21 @@ import jakarta.annotation.Nonnull; import java.net.URI; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.polaris.core.storage.azure.AzureLocation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** An abstraction over a storage location */ public class StorageLocation { - private final String location; + private static final Logger LOGGER = LoggerFactory.getLogger(StorageLocation.class); + private static final Pattern SCHEME_PATTERN = Pattern.compile("^(.+?):(.+)"); + public static final String LOCAL_PATH_PREFIX = "file:///"; + private final String location; + /** Create a StorageLocation from a String path */ public static StorageLocation of(String location) { // TODO implement StorageLocation for all supported file systems and add isValidLocation @@ -99,4 +107,20 @@ public boolean isChildOf(StorageLocation potentialParent) { return slashTerminatedLocation.startsWith(slashTerminatedParentLocation); } } + + /** Returns a string representation of the location but without a scheme */ + public String withoutScheme() { + if (location == null) { + return null; + } + Matcher matcher = SCHEME_PATTERN.matcher(location); + if (matcher.matches()) { + String locationWithoutScheme = matcher.group(2); + LOGGER.debug("Extracted {} from location {}", locationWithoutScheme, location); + return locationWithoutScheme; + } else { + LOGGER.debug("Found no scheme in location {}", location); + return location; + } + } } diff --git a/runtime/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java b/runtime/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java index dca58933cb..3d57024b7e 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java @@ -213,6 +213,8 @@ public Map getConfigOverrides() { "polaris.event-listener.type", "test", "polaris.readiness.ignore-severe-issues", + "true", + "polaris.features.\"ALLOW_TABLE_LOCATION_OVERLAP\"", "true"); } } diff --git a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java index 1a1914ad10..946cad71cb 100644 --- a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java +++ b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java @@ -576,7 +576,7 @@ private void authorizeGrantOnPolicyOperationOrThrow( /** Get all locations where data for a `CatalogEntity` may be stored */ private Set getCatalogLocations(CatalogEntity catalogEntity) { HashSet catalogLocations = new HashSet<>(); - catalogLocations.add(terminateWithSlash(catalogEntity.getDefaultBaseLocation())); + catalogLocations.add(terminateWithSlash(catalogEntity.getBaseLocation())); if (catalogEntity.getStorageConfigurationInfo() != null) { catalogLocations.addAll( catalogEntity.getStorageConfigurationInfo().getAllowedLocations().stream() @@ -872,7 +872,7 @@ private void validateUpdateCatalogDiffOrThrow( } CatalogEntity.Builder updateBuilder = new CatalogEntity.Builder(currentCatalogEntity); - String defaultBaseLocation = currentCatalogEntity.getDefaultBaseLocation(); + String defaultBaseLocation = currentCatalogEntity.getBaseLocation(); if (updateRequest.getProperties() != null) { Map updateProperties = reservedProperties.removeReservedPropertiesFromUpdate( diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index e99513155a..f8b7edf337 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -93,6 +93,7 @@ import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.core.entity.LocationBasedEntity; import org.apache.polaris.core.entity.NamespaceEntity; import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntityConstants; @@ -237,7 +238,7 @@ public void initialize(String name, Map properties) { // Base location from catalogEntity is primary source of truth, otherwise fall through // to the same key from the properties map, and finally fall through to WAREHOUSE_LOCATION. String baseLocation = - Optional.ofNullable(catalogEntity.getDefaultBaseLocation()) + Optional.ofNullable(catalogEntity.getBaseLocation()) .orElse( properties.getOrDefault( CatalogEntity.DEFAULT_BASE_LOCATION_KEY, @@ -507,6 +508,18 @@ private void createNamespaceInternal( Map metadata, PolarisResolvedPathWrapper resolvedParent) { String baseLocation = resolveNamespaceLocation(namespace, metadata); + + // Set / suffix + boolean requireTrailingSlash = + callContext + .getPolarisCallContext() + .getConfigurationStore() + .getConfiguration( + callContext.getRealmContext(), FeatureConfiguration.ADD_TRAILING_SLASH_TO_LOCATION); + if (requireTrailingSlash && !baseLocation.endsWith("/")) { + baseLocation += "/"; + } + NamespaceEntity entity = new NamespaceEntity.Builder(namespace) .setCatalogId(getCatalogId()) @@ -522,8 +535,7 @@ private void createNamespaceInternal( .getConfiguration( callContext.getRealmContext(), FeatureConfiguration.ALLOW_NAMESPACE_LOCATION_OVERLAP)) { LOGGER.debug("Validating no overlap for {} with sibling tables or namespaces", namespace); - validateNoLocationOverlap( - entity.getBaseLocation(), resolvedParent.getRawFullPath(), entity.getName()); + validateNoLocationOverlap(entity, resolvedParent.getRawFullPath()); } else { LOGGER.debug("Skipping location overlap validation for namespace '{}'", namespace); } @@ -578,7 +590,7 @@ private String resolveNamespaceLocation(Namespace namespace, Map @Nonnull CallContext callContext, PolarisEntity entity) { if (entity.getType().equals(PolarisEntityType.CATALOG)) { CatalogEntity catEntity = CatalogEntity.of(entity); - String catalogDefaultBaseLocation = catEntity.getDefaultBaseLocation(); + String catalogDefaultBaseLocation = catEntity.getBaseLocation(); callContext .getPolarisCallContext() .getDiagServices() @@ -695,9 +707,7 @@ public boolean setProperties(Namespace namespace, Map properties callContext.getRealmContext(), FeatureConfiguration.ALLOW_NAMESPACE_LOCATION_OVERLAP)) { LOGGER.debug("Validating no overlap with sibling tables or namespaces"); validateNoLocationOverlap( - NamespaceEntity.of(updatedEntity).getBaseLocation(), - resolvedEntities.getRawParentPath(), - updatedEntity.getName()); + NamespaceEntity.of(updatedEntity), resolvedEntities.getRawParentPath()); } else { LOGGER.debug("Skipping location overlap validation for namespace '{}'", namespace); } @@ -1038,7 +1048,17 @@ private void validateNoLocationOverlap( } else if (validateViewOverlap || entity.getSubType().equals(PolarisEntitySubType.ICEBERG_TABLE)) { LOGGER.debug("Validating no overlap with sibling tables or namespaces"); - validateNoLocationOverlap(location, resolvedNamespace, identifier.name()); + + // Create a fake IcebergTableLikeEntity to check for overlap, since no real entity + // has been created yet. + IcebergTableLikeEntity virtualEntity = + IcebergTableLikeEntity.of( + new PolarisEntity.Builder() + .setParentId(resolvedNamespace.getLast().getId()) + .setProperties(Map.of(PolarisEntityConstants.ENTITY_BASE_LOCATION, location)) + .build()); + + validateNoLocationOverlap(virtualEntity, resolvedNamespace); } } @@ -1049,8 +1069,41 @@ private void validateNoLocationOverlap( * base-location property of each. The target entity's base location may not be a prefix or a * suffix of any sibling entity's base location. */ - private void validateNoLocationOverlap( - String location, List parentPath, String name) { + private void validateNoLocationOverlap( + T entity, List parentPath) { + + String location = entity.getBaseLocation(); + String name = entity.getName(); + + // Attempt to directly query for siblings + boolean useOptimizedSiblingCheck = + callContext + .getPolarisCallContext() + .getConfigurationStore() + .getConfiguration( + callContext.getRealmContext(), FeatureConfiguration.OPTIMIZED_SIBLING_CHECK); + if (useOptimizedSiblingCheck) { + Optional> directSiblingCheckResult = + getMetaStoreManager().hasOverlappingSiblings(callContext.getPolarisCallContext(), entity); + if (directSiblingCheckResult.isPresent()) { + if (directSiblingCheckResult.get().isPresent()) { + throw new org.apache.iceberg.exceptions.ForbiddenException( + "Unable to create entity at location '%s' because it conflicts with existing table or namespace at %s", + location, directSiblingCheckResult.get().get()); + } else { + return; + } + } + } + + // if the entity path has more than just the catalog, check for tables as well as other + // namespaces + Optional parentNamespace = + parentPath.size() > 1 + ? Optional.of(NamespaceEntity.of(parentPath.getLast())) + : Optional.empty(); + + // Fall through by listing everything: ListEntitiesResult siblingNamespacesResult = getMetaStoreManager() .listEntities( @@ -1064,13 +1117,6 @@ private void validateNoLocationOverlap( "Unable to resolve siblings entities to validate location - could not list namespaces"); } - // if the entity path has more than just the catalog, check for tables as well as other - // namespaces - Optional parentNamespace = - parentPath.size() > 1 - ? Optional.of(NamespaceEntity.of(parentPath.getLast())) - : Optional.empty(); - List siblingTables = parentNamespace .map( @@ -2167,26 +2213,47 @@ private void createTableLike(TableIdentifier identifier, PolarisEntity entity) { private void createTableLike( TableIdentifier identifier, PolarisEntity entity, PolarisResolvedPathWrapper resolvedParent) { + IcebergTableLikeEntity icebergTableLikeEntity = IcebergTableLikeEntity.of(entity); + // Set / suffix + boolean requireTrailingSlash = + callContext + .getPolarisCallContext() + .getConfigurationStore() + .getConfiguration( + callContext.getRealmContext(), FeatureConfiguration.ADD_TRAILING_SLASH_TO_LOCATION); + if (requireTrailingSlash + && icebergTableLikeEntity.getBaseLocation() != null + && !icebergTableLikeEntity.getBaseLocation().endsWith("/")) { + icebergTableLikeEntity = + new IcebergTableLikeEntity.Builder(icebergTableLikeEntity) + .setBaseLocation(icebergTableLikeEntity.getBaseLocation() + "/") + .build(); + } + // Make sure the metadata file is valid for our allowed locations. - String metadataLocation = IcebergTableLikeEntity.of(entity).getMetadataLocation(); + String metadataLocation = icebergTableLikeEntity.getMetadataLocation(); validateLocationForTableLike(identifier, metadataLocation, resolvedParent); List catalogPath = resolvedParent.getRawFullPath(); - if (entity.getParentId() <= 0) { + if (icebergTableLikeEntity.getParentId() <= 0) { // TODO: Validate catalogPath size is at least 1 for catalog entity? - entity = - new PolarisEntity.Builder(entity) + icebergTableLikeEntity = + new IcebergTableLikeEntity.Builder(icebergTableLikeEntity) .setParentId(resolvedParent.getRawLeafEntity().getId()) .build(); } - entity = - new PolarisEntity.Builder(entity).setCreateTimestamp(System.currentTimeMillis()).build(); + icebergTableLikeEntity = + new IcebergTableLikeEntity.Builder(icebergTableLikeEntity) + .setCreateTimestamp(System.currentTimeMillis()) + .build(); EntityResult res = getMetaStoreManager() .createEntityIfNotExists( - getCurrentPolarisContext(), PolarisEntity.toCoreList(catalogPath), entity); + getCurrentPolarisContext(), + PolarisEntity.toCoreList(catalogPath), + icebergTableLikeEntity); if (!res.isSuccess()) { switch (res.getReturnStatus()) { case BaseResult.ReturnStatus.CATALOG_PATH_CANNOT_BE_RESOLVED: @@ -2214,16 +2281,35 @@ private void updateTableLike(TableIdentifier identifier, PolarisEntity entity) { throw new IllegalStateException( String.format("Failed to fetch resolved TableIdentifier '%s'", identifier)); } + IcebergTableLikeEntity icebergTableLikeEntity = new IcebergTableLikeEntity(entity); + + // Set / suffix + boolean requireTrailingSlash = + callContext + .getPolarisCallContext() + .getConfigurationStore() + .getConfiguration( + callContext.getRealmContext(), FeatureConfiguration.ADD_TRAILING_SLASH_TO_LOCATION); + if (requireTrailingSlash + && icebergTableLikeEntity.getBaseLocation() != null + && !icebergTableLikeEntity.getBaseLocation().endsWith("/")) { + icebergTableLikeEntity = + new IcebergTableLikeEntity.Builder(icebergTableLikeEntity) + .setBaseLocation(icebergTableLikeEntity.getBaseLocation() + "/") + .build(); + } // Make sure the metadata file is valid for our allowed locations. - String metadataLocation = IcebergTableLikeEntity.of(entity).getMetadataLocation(); + String metadataLocation = icebergTableLikeEntity.getMetadataLocation(); validateLocationForTableLike(identifier, metadataLocation, resolvedEntities); List catalogPath = resolvedEntities.getRawParentPath(); EntityResult res = getMetaStoreManager() .updateEntityPropertiesIfNotChanged( - getCurrentPolarisContext(), PolarisEntity.toCoreList(catalogPath), entity); + getCurrentPolarisContext(), + PolarisEntity.toCoreList(catalogPath), + icebergTableLikeEntity); if (!res.isSuccess()) { switch (res.getReturnStatus()) { case BaseResult.ReturnStatus.CATALOG_PATH_CANNOT_BE_RESOLVED: @@ -2489,7 +2575,7 @@ private Page listTableLike( protected FileIO loadFileIO(String ioImpl, Map properties) { IcebergTableLikeEntity icebergTableLikeEntity = IcebergTableLikeEntity.of(catalogEntity); TableIdentifier identifier = icebergTableLikeEntity.getTableIdentifier(); - Set locations = Set.of(catalogEntity.getDefaultBaseLocation()); + Set locations = Set.of(catalogEntity.getBaseLocation()); ResolvedPolarisEntity resolvedCatalogEntity = new ResolvedPolarisEntity(catalogEntity, List.of(), List.of()); PolarisResolvedPathWrapper resolvedPath = diff --git a/service/common/src/main/java/org/apache/polaris/service/context/catalog/PolarisCallContextCatalogFactory.java b/service/common/src/main/java/org/apache/polaris/service/context/catalog/PolarisCallContextCatalogFactory.java index 5ac255821c..60c505d42c 100644 --- a/service/common/src/main/java/org/apache/polaris/service/context/catalog/PolarisCallContextCatalogFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/context/catalog/PolarisCallContextCatalogFactory.java @@ -99,7 +99,7 @@ public Catalog createCallContextCatalog( CatalogEntity catalog = CatalogEntity.of(baseCatalogEntity); Map catalogProperties = new HashMap<>(catalog.getPropertiesAsMap()); - String defaultBaseLocation = catalog.getDefaultBaseLocation(); + String defaultBaseLocation = catalog.getBaseLocation(); LOGGER.debug( "Looked up defaultBaseLocation {} for catalog {}", defaultBaseLocation, catalogKey);