From 6c90a3d6dfaec73f2e979c8be0a1d4169b3c6c10 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 6 Nov 2024 11:45:26 -0800 Subject: [PATCH 01/88] wip --- .../polaris/service/catalog/BasePolarisCatalog.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 3d926fb002..9e9ed37af8 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -811,6 +811,17 @@ public Map getCredentialConfig( storageInfo.get()); } + @Override + public Table loadTable(TableIdentifier identifier) { + // #### + boolean allowMetadata = callContext + .getPolarisCallContext() + .getConfigurationStore() + .getConfiguration( + callContext.getPolarisCallContext(), + PolarisConfiguration.SUPPORTED_CATALOG_STORAGE_TYPES); + } + /** * Based on configuration settings, for callsites that need to handle potentially setting a new * base location for a TableLike entity, produces the transformed location if applicable, or else From bda008ee95cd8003b739f23bf71b5d564c56f8c0 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 6 Nov 2024 16:13:54 -0800 Subject: [PATCH 02/88] first real commit --- .../polaris/core/PolarisConfiguration.java | 8 ++ .../core/entity/PolarisEntityType.java | 3 +- .../core/entity/TableMetadataEntity.java | 68 +++++++++++++ .../PolarisMetaStoreManagerImpl.java | 3 + .../service/catalog/BasePolarisCatalog.java | 28 +++++- .../persistence/MetadataCacheManager.java | 95 +++++++++++++++++++ 6 files changed, 199 insertions(+), 6 deletions(-) create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java create mode 100644 polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java diff --git a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java index 397c9afd9d..6266cbcbfb 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java @@ -191,4 +191,12 @@ public static Builder builder() { "If set to true, allows tables to be dropped with the purge parameter set to true.") .defaultValue(true) .build(); + + public static final PolarisConfiguration USE_METADATA_CACHE = + PolarisConfiguration.builder() + .key("USE_METADATA_CACHE") + .description( + "If set to true, support serving table metadata without reading the metadata.json file.") + .defaultValue(false) + .build(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityType.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityType.java index fa3a2d6618..308dcb8186 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityType.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityType.java @@ -34,7 +34,8 @@ public enum PolarisEntityType { // generic table is either a view or a real table TABLE_LIKE(7, NAMESPACE, false, false), TASK(8, ROOT, false, false), - FILE(9, TABLE_LIKE, false, false); + FILE(9, TABLE_LIKE, false, false), + TABLE_METADATA(10, TABLE_LIKE, false, false); // to efficiently map a code to its corresponding entity type, use a reverse array which // is initialized below diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java new file mode 100644 index 0000000000..6afd4f118b --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.entity; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +public class TableMetadataEntity extends PolarisEntity { + private static String CONTENT_KEY = "content"; + private static String METADATA_LOCATION_KEY = "metadata_location"; + + public TableMetadataEntity(PolarisBaseEntity sourceEntity) { + super(sourceEntity); + } + + public static TableMetadataEntity of(PolarisBaseEntity sourceEntity) { + if (sourceEntity != null) { + return new TableMetadataEntity(sourceEntity); + } + return null; + } + + @JsonIgnore + public String getContent() { + return getInternalPropertiesAsMap().get(CONTENT_KEY); + } + + @JsonIgnore + public String getMetadataLocation() { + return getInternalPropertiesAsMap().get(METADATA_LOCATION_KEY); + } + + public static class Builder extends PolarisEntity.BaseBuilder { + public Builder(String metadataLocation, String content) { + this.internalProperties.put(CONTENT_KEY, content); + } + + @Override + public TableMetadataEntity build() { + return new TableMetadataEntity(buildBase()); + } + + public TableMetadataEntity.Builder setContent(String content) { + this.internalProperties.put(CONTENT_KEY, content); + return this; + } + + public TableMetadataEntity.Builder setMetadataLocation(String metadataLocation) { + this.internalProperties.put(METADATA_LOCATION_KEY, metadataLocation); + return this; + } + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index b05129fa3d..2095ab33b1 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -49,6 +50,8 @@ import org.apache.polaris.core.entity.PolarisPrincipalSecrets; import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.entity.PolarisTaskConstants; +import org.apache.polaris.core.entity.TableMetadataEntity; +import org.apache.polaris.core.persistence.models.ModelEntityActive; import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 9e9ed37af8..8dc2358c8e 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -37,6 +37,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Predicate; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.lang3.exception.ExceptionUtils; @@ -99,6 +100,7 @@ import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageIntegration; import org.apache.polaris.core.storage.aws.PolarisS3FileIOClientFactory; +import org.apache.polaris.service.persistence.MetadataCacheManager; import org.apache.polaris.service.task.TaskExecutor; import org.apache.polaris.service.types.NotificationRequest; import org.apache.polaris.service.types.NotificationType; @@ -811,15 +813,31 @@ public Map getCredentialConfig( storageInfo.get()); } - @Override - public Table loadTable(TableIdentifier identifier) { - // #### - boolean allowMetadata = callContext + public TableMetadata loadTableMetadata(TableIdentifier identifier) { + boolean useMetadataCache = callContext .getPolarisCallContext() .getConfigurationStore() .getConfiguration( callContext.getPolarisCallContext(), - PolarisConfiguration.SUPPORTED_CATALOG_STORAGE_TYPES); + PolarisConfiguration.USE_METADATA_CACHE); + if (!useMetadataCache) { + return loadTableMetadata(loadTable(identifier)); + } else { + Supplier fallback = () -> loadTableMetadata(loadTable(identifier)); + return MetadataCacheManager.loadTableMetadata( + identifier, + callContext.getPolarisCallContext(), + entityManager, + resolvedEntityView, + fallback); + } + } + + private static TableMetadata loadTableMetadata(Table table) { + if (table instanceof BaseTable baseTable) { + return baseTable.operations().current(); + } + throw new IllegalArgumentException("Cannot load metadata for " + table.name()); } /** diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java new file mode 100644 index 0000000000..5aaa35f24f --- /dev/null +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.persistence; + +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.hadoop.HadoopTableOperations; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.TableLikeEntity; +import org.apache.polaris.core.entity.TableMetadataEntity; +import org.apache.polaris.core.persistence.PolarisEntityManager; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; +import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView; +import org.jetbrains.annotations.NotNull; + +import java.util.List; +import java.util.Optional; +import java.util.function.Function; +import java.util.function.Supplier; + +public class MetadataCacheManager { + + /** + * Load the cached {@link Table} or fall back to `fallback` if one doesn't exist + */ + public static TableMetadata loadTableMetadata( + TableIdentifier tableIdentifier, + PolarisCallContext callContext, + PolarisEntityManager entityManager, + PolarisResolutionManifestCatalogView resolvedEntityView, + Supplier fallback) { + // TODO add write to cache + return loadCachedTableMetadata( + tableIdentifier, callContext, entityManager, resolvedEntityView) + .orElseGet(fallback); + } + + /** + * Return the cached {@link Table} entity, if one exists + */ + private static @NotNull Optional loadCachedTableMetadata( + TableIdentifier tableIdentifier, + PolarisCallContext callContext, + PolarisEntityManager entityManager, + PolarisResolutionManifestCatalogView resolvedEntityView) { + PolarisResolvedPathWrapper resolvedEntities = + resolvedEntityView.getPassthroughResolvedPath( + tableIdentifier, PolarisEntitySubType.TABLE); + if (resolvedEntities == null) { + return Optional.empty(); + } else { + TableLikeEntity entity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); + String metadataLocation = entity.getMetadataLocation(); + PolarisMetaStoreManager.EntityResult metadataEntityResult = entityManager + .getMetaStoreManager() + .readEntityByName( + callContext, + resolvedEntities.getRawFullPath().stream().map(PolarisEntityCore::new).toList(), + PolarisEntityType.TABLE_METADATA, + PolarisEntitySubType.ANY_SUBTYPE, + metadataLocation); + return Optional + .ofNullable(metadataEntityResult.getEntity()) + .map(metadataEntity -> { + TableMetadataEntity tableMetadataEntity = (TableMetadataEntity) metadataEntity; + return TableMetadataParser.fromJson(tableMetadataEntity.getContent()); + }); + } + } +} From c38a01df46d46049a492ac39682a93b1fd7846cc Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 6 Nov 2024 16:38:02 -0800 Subject: [PATCH 03/88] cleaning up --- .../persistence/MetadataCacheManager.java | 61 +++++++++++++++++-- 1 file changed, 56 insertions(+), 5 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 5aaa35f24f..ea2aa1061d 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -27,6 +27,7 @@ import org.apache.iceberg.hadoop.HadoopTableOperations; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; @@ -36,7 +37,10 @@ import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView; +import org.apache.polaris.service.auth.TestOAuth2ApiService; import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.List; import java.util.Optional; @@ -44,6 +48,7 @@ import java.util.function.Supplier; public class MetadataCacheManager { + private static final Logger LOGGER = LoggerFactory.getLogger(MetadataCacheManager.class); /** * Load the cached {@link Table} or fall back to `fallback` if one doesn't exist @@ -54,10 +59,56 @@ public static TableMetadata loadTableMetadata( PolarisEntityManager entityManager, PolarisResolutionManifestCatalogView resolvedEntityView, Supplier fallback) { - // TODO add write to cache - return loadCachedTableMetadata( - tableIdentifier, callContext, entityManager, resolvedEntityView) - .orElseGet(fallback); + LOGGER.debug(String.format("Loading cached metadata for %s", tableIdentifier)); + Optional cachedMetadata = loadCachedTableMetadata( + tableIdentifier, callContext, entityManager, resolvedEntityView); + if (cachedMetadata.isPresent()) { + LOGGER.debug(String.format("Using cached metadata for %s", tableIdentifier)); + return cachedMetadata.get(); + } else { + TableMetadata metadata = fallback.get(); + PolarisMetaStoreManager.EntityResult cacheResult = cacheTableMetadata( + tableIdentifier, + metadata, + callContext, + entityManager, + resolvedEntityView); + if (!cacheResult.isSuccess()) { + LOGGER.debug(String.format("Failed to cache metadata for %s", tableIdentifier)); + } + return metadata; + } + } + + /** + * Attempt to add table metadata to the cache + * @return The result of trying to cache the metadata + */ + private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( + TableIdentifier tableIdentifier, + TableMetadata metadata, + PolarisCallContext callContext, + PolarisEntityManager entityManager, + PolarisResolutionManifestCatalogView resolvedEntityView) { + PolarisResolvedPathWrapper resolvedEntities = + resolvedEntityView.getPassthroughResolvedPath( + tableIdentifier, PolarisEntitySubType.TABLE); + if (resolvedEntities == null) { + return new PolarisMetaStoreManager.EntityResult( + PolarisMetaStoreManager.ReturnStatus.ENTITY_NOT_FOUND, null); + } else { + TableLikeEntity tableEntity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); + TableMetadataEntity tableMetadataEntity = + new TableMetadataEntity.Builder(metadata.location(), TableMetadataParser.toJson(metadata)) + .setParentId(tableEntity.getId()) + .build(); + return entityManager + .getMetaStoreManager() + .createEntityIfNotExists( + callContext, + PolarisEntity.toCoreList(resolvedEntities.getRawFullPath()), + tableMetadataEntity); + } } /** @@ -80,7 +131,7 @@ public static TableMetadata loadTableMetadata( .getMetaStoreManager() .readEntityByName( callContext, - resolvedEntities.getRawFullPath().stream().map(PolarisEntityCore::new).toList(), + PolarisEntity.toCoreList(resolvedEntities.getRawFullPath()), PolarisEntityType.TABLE_METADATA, PolarisEntitySubType.ANY_SUBTYPE, metadataLocation); From 3a699531029c1fdc4a4f8b533ef8a483d673c89e Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 6 Nov 2024 16:46:19 -0800 Subject: [PATCH 04/88] tests fixed --- .../org/apache/polaris/core/entity/TableMetadataEntity.java | 6 +++++- .../polaris/service/persistence/MetadataCacheManager.java | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java index 6afd4f118b..13aa28ca47 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java @@ -47,7 +47,11 @@ public String getMetadataLocation() { public static class Builder extends PolarisEntity.BaseBuilder { public Builder(String metadataLocation, String content) { - this.internalProperties.put(CONTENT_KEY, content); + super(); + setName(metadataLocation); + setType(PolarisEntityType.TABLE_METADATA); + setMetadataLocation(metadataLocation); + setContent(content); } @Override diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index ea2aa1061d..0692d6cbf5 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -100,7 +100,9 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( TableLikeEntity tableEntity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); TableMetadataEntity tableMetadataEntity = new TableMetadataEntity.Builder(metadata.location(), TableMetadataParser.toJson(metadata)) + .setCatalogId(tableEntity.getCatalogId()) .setParentId(tableEntity.getId()) + .setId(entityManager.getMetaStoreManager().generateNewEntityId(callContext).getId()) .build(); return entityManager .getMetaStoreManager() From ca856edb49e9a0ad695507ebc2cb8248bf095963 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 6 Nov 2024 16:50:32 -0800 Subject: [PATCH 05/88] begin refactoring --- .../catalog/PolarisCatalogHandlerWrapper.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java index 29aa328c73..26b361df93 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java @@ -822,10 +822,21 @@ public LoadTableResponse loadTableWithAccessDelegation( // when data-access is specified but access delegation grants are not found. return doCatalogOperation( () -> { - Table table = baseCatalog.loadTable(tableIdentifier); + final TableMetadata tableMetadata; + if (baseCatalog instanceof BasePolarisCatalog basePolarisCatalog) { + tableMetadata = basePolarisCatalog.loadTableMetadata(tableIdentifier); + } else { + // TODO fix + Table table = baseCatalog.loadTable(tableIdentifier); + if (table instanceof BaseTable baseTable) { + tableMetadata = baseTable.operations().current(); + } else { + tableMetadata = null; + } + } + if (table instanceof BaseTable baseTable) { - TableMetadata tableMetadata = baseTable.operations().current(); LoadTableResponse.Builder responseBuilder = LoadTableResponse.builder().withTableMetadata(tableMetadata); if (baseCatalog instanceof SupportsCredentialDelegation credentialDelegation) { From 779d9122d5fee43379085e1f5270e4c7a5d06871 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 6 Nov 2024 17:02:49 -0800 Subject: [PATCH 06/88] wire up to loadTable --- .../catalog/PolarisCatalogHandlerWrapper.java | 51 +++++++++---------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java index 26b361df93..2c0a9eb30d 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java @@ -792,7 +792,15 @@ public LoadTableResponse loadTable(TableIdentifier tableIdentifier, String snaps PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LOAD_TABLE; authorizeBasicTableLikeOperationOrThrow(op, PolarisEntitySubType.TABLE, tableIdentifier); - return doCatalogOperation(() -> CatalogHandlers.loadTable(baseCatalog, tableIdentifier)); + return doCatalogOperation(() -> { + if (baseCatalog instanceof BasePolarisCatalog basePolarisCatalog) { + return LoadTableResponse.builder() + .withTableMetadata(basePolarisCatalog.loadTableMetadata(tableIdentifier)) + .build(); + } + + return CatalogHandlers.loadTable(baseCatalog, tableIdentifier); + }); } public LoadTableResponse loadTableWithAccessDelegation( @@ -822,40 +830,31 @@ public LoadTableResponse loadTableWithAccessDelegation( // when data-access is specified but access delegation grants are not found. return doCatalogOperation( () -> { - final TableMetadata tableMetadata; + TableMetadata tableMetadata = null; if (baseCatalog instanceof BasePolarisCatalog basePolarisCatalog) { tableMetadata = basePolarisCatalog.loadTableMetadata(tableIdentifier); - } else { - // TODO fix - Table table = baseCatalog.loadTable(tableIdentifier); - if (table instanceof BaseTable baseTable) { - tableMetadata = baseTable.operations().current(); - } else { - tableMetadata = null; - } } + // The metadata failed to load + if (tableMetadata == null) { + throw new NoSuchTableException("Table does not exist: %s", tableIdentifier.toString()); + } - if (table instanceof BaseTable baseTable) { + if (baseCatalog instanceof SupportsCredentialDelegation credentialDelegation) { LoadTableResponse.Builder responseBuilder = LoadTableResponse.builder().withTableMetadata(tableMetadata); - if (baseCatalog instanceof SupportsCredentialDelegation credentialDelegation) { - LOGGER - .atDebug() - .addKeyValue("tableIdentifier", tableIdentifier) - .addKeyValue("tableLocation", tableMetadata.location()) - .log("Fetching client credentials for table"); - responseBuilder.addAllConfig( - credentialDelegation.getCredentialConfig( - tableIdentifier, tableMetadata, actionsRequested)); - } + LOGGER + .atDebug() + .addKeyValue("tableIdentifier", tableIdentifier) + .addKeyValue("tableLocation", tableMetadata.location()) + .log("Fetching client credentials for table"); + responseBuilder.addAllConfig( + credentialDelegation.getCredentialConfig( + tableIdentifier, tableMetadata, actionsRequested)); return responseBuilder.build(); - } else if (table instanceof BaseMetadataTable) { - // metadata tables are loaded on the client side, return NoSuchTableException for now - throw new NoSuchTableException("Table does not exist: %s", tableIdentifier.toString()); + } else { + throw new IllegalStateException("Cannot wrap catalog that does not produce BaseTable"); } - - throw new IllegalStateException("Cannot wrap catalog that does not produce BaseTable"); }); } From b4c533546cb83a5f43b4446c34203a2aeb3ca67d Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 6 Nov 2024 20:49:15 -0800 Subject: [PATCH 07/88] autolint --- .../core/entity/TableMetadataEntity.java | 77 +++---- .../PolarisMetaStoreManagerImpl.java | 3 - .../service/catalog/BasePolarisCatalog.java | 12 +- .../catalog/PolarisCatalogHandlerWrapper.java | 17 +- .../persistence/MetadataCacheManager.java | 188 ++++++++---------- 5 files changed, 140 insertions(+), 157 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java index 13aa28ca47..ae12ded449 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java @@ -21,52 +21,53 @@ import com.fasterxml.jackson.annotation.JsonIgnore; public class TableMetadataEntity extends PolarisEntity { - private static String CONTENT_KEY = "content"; - private static String METADATA_LOCATION_KEY = "metadata_location"; + private static String CONTENT_KEY = "content"; + private static String METADATA_LOCATION_KEY = "metadata_location"; - public TableMetadataEntity(PolarisBaseEntity sourceEntity) { - super(sourceEntity); - } + public TableMetadataEntity(PolarisBaseEntity sourceEntity) { + super(sourceEntity); + } - public static TableMetadataEntity of(PolarisBaseEntity sourceEntity) { - if (sourceEntity != null) { - return new TableMetadataEntity(sourceEntity); - } - return null; + public static TableMetadataEntity of(PolarisBaseEntity sourceEntity) { + if (sourceEntity != null) { + return new TableMetadataEntity(sourceEntity); } + return null; + } - @JsonIgnore - public String getContent() { - return getInternalPropertiesAsMap().get(CONTENT_KEY); - } + @JsonIgnore + public String getContent() { + return getInternalPropertiesAsMap().get(CONTENT_KEY); + } - @JsonIgnore - public String getMetadataLocation() { - return getInternalPropertiesAsMap().get(METADATA_LOCATION_KEY); - } + @JsonIgnore + public String getMetadataLocation() { + return getInternalPropertiesAsMap().get(METADATA_LOCATION_KEY); + } - public static class Builder extends PolarisEntity.BaseBuilder { - public Builder(String metadataLocation, String content) { - super(); - setName(metadataLocation); - setType(PolarisEntityType.TABLE_METADATA); - setMetadataLocation(metadataLocation); - setContent(content); - } + public static class Builder + extends PolarisEntity.BaseBuilder { + public Builder(String metadataLocation, String content) { + super(); + setName(metadataLocation); + setType(PolarisEntityType.TABLE_METADATA); + setMetadataLocation(metadataLocation); + setContent(content); + } - @Override - public TableMetadataEntity build() { - return new TableMetadataEntity(buildBase()); - } + @Override + public TableMetadataEntity build() { + return new TableMetadataEntity(buildBase()); + } - public TableMetadataEntity.Builder setContent(String content) { - this.internalProperties.put(CONTENT_KEY, content); - return this; - } + public TableMetadataEntity.Builder setContent(String content) { + this.internalProperties.put(CONTENT_KEY, content); + return this; + } - public TableMetadataEntity.Builder setMetadataLocation(String metadataLocation) { - this.internalProperties.put(METADATA_LOCATION_KEY, metadataLocation); - return this; - } + public TableMetadataEntity.Builder setMetadataLocation(String metadataLocation) { + this.internalProperties.put(METADATA_LOCATION_KEY, metadataLocation); + return this; } + } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index 2095ab33b1..b05129fa3d 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -30,7 +30,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -50,8 +49,6 @@ import org.apache.polaris.core.entity.PolarisPrincipalSecrets; import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.entity.PolarisTaskConstants; -import org.apache.polaris.core.entity.TableMetadataEntity; -import org.apache.polaris.core.persistence.models.ModelEntityActive; import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 8dc2358c8e..c2462a11e3 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -814,12 +814,12 @@ public Map getCredentialConfig( } public TableMetadata loadTableMetadata(TableIdentifier identifier) { - boolean useMetadataCache = callContext - .getPolarisCallContext() - .getConfigurationStore() - .getConfiguration( - callContext.getPolarisCallContext(), - PolarisConfiguration.USE_METADATA_CACHE); + boolean useMetadataCache = + callContext + .getPolarisCallContext() + .getConfigurationStore() + .getConfiguration( + callContext.getPolarisCallContext(), PolarisConfiguration.USE_METADATA_CACHE); if (!useMetadataCache) { return loadTableMetadata(loadTable(identifier)); } else { diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java index 2c0a9eb30d..6e41168c5c 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java @@ -792,15 +792,16 @@ public LoadTableResponse loadTable(TableIdentifier tableIdentifier, String snaps PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LOAD_TABLE; authorizeBasicTableLikeOperationOrThrow(op, PolarisEntitySubType.TABLE, tableIdentifier); - return doCatalogOperation(() -> { - if (baseCatalog instanceof BasePolarisCatalog basePolarisCatalog) { - return LoadTableResponse.builder() - .withTableMetadata(basePolarisCatalog.loadTableMetadata(tableIdentifier)) - .build(); - } + return doCatalogOperation( + () -> { + if (baseCatalog instanceof BasePolarisCatalog basePolarisCatalog) { + return LoadTableResponse.builder() + .withTableMetadata(basePolarisCatalog.loadTableMetadata(tableIdentifier)) + .build(); + } - return CatalogHandlers.loadTable(baseCatalog, tableIdentifier); - }); + return CatalogHandlers.loadTable(baseCatalog, tableIdentifier); + }); } public LoadTableResponse loadTableWithAccessDelegation( diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 0692d6cbf5..c94b80a234 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -16,19 +16,16 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.polaris.service.persistence; +import java.util.Optional; +import java.util.function.Supplier; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableMetadataParser; -import org.apache.iceberg.TableOperations; import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.hadoop.HadoopTableOperations; import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntity; -import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.TableLikeEntity; @@ -37,112 +34,99 @@ import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView; -import org.apache.polaris.service.auth.TestOAuth2ApiService; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.List; -import java.util.Optional; -import java.util.function.Function; -import java.util.function.Supplier; - public class MetadataCacheManager { - private static final Logger LOGGER = LoggerFactory.getLogger(MetadataCacheManager.class); + private static final Logger LOGGER = LoggerFactory.getLogger(MetadataCacheManager.class); - /** - * Load the cached {@link Table} or fall back to `fallback` if one doesn't exist - */ - public static TableMetadata loadTableMetadata( - TableIdentifier tableIdentifier, - PolarisCallContext callContext, - PolarisEntityManager entityManager, - PolarisResolutionManifestCatalogView resolvedEntityView, - Supplier fallback) { - LOGGER.debug(String.format("Loading cached metadata for %s", tableIdentifier)); - Optional cachedMetadata = loadCachedTableMetadata( - tableIdentifier, callContext, entityManager, resolvedEntityView); - if (cachedMetadata.isPresent()) { - LOGGER.debug(String.format("Using cached metadata for %s", tableIdentifier)); - return cachedMetadata.get(); - } else { - TableMetadata metadata = fallback.get(); - PolarisMetaStoreManager.EntityResult cacheResult = cacheTableMetadata( - tableIdentifier, - metadata, - callContext, - entityManager, - resolvedEntityView); - if (!cacheResult.isSuccess()) { - LOGGER.debug(String.format("Failed to cache metadata for %s", tableIdentifier)); - } - return metadata; - } + /** Load the cached {@link Table} or fall back to `fallback` if one doesn't exist */ + public static TableMetadata loadTableMetadata( + TableIdentifier tableIdentifier, + PolarisCallContext callContext, + PolarisEntityManager entityManager, + PolarisResolutionManifestCatalogView resolvedEntityView, + Supplier fallback) { + LOGGER.debug(String.format("Loading cached metadata for %s", tableIdentifier)); + Optional cachedMetadata = + loadCachedTableMetadata(tableIdentifier, callContext, entityManager, resolvedEntityView); + if (cachedMetadata.isPresent()) { + LOGGER.debug(String.format("Using cached metadata for %s", tableIdentifier)); + return cachedMetadata.get(); + } else { + TableMetadata metadata = fallback.get(); + PolarisMetaStoreManager.EntityResult cacheResult = + cacheTableMetadata( + tableIdentifier, metadata, callContext, entityManager, resolvedEntityView); + if (!cacheResult.isSuccess()) { + LOGGER.debug(String.format("Failed to cache metadata for %s", tableIdentifier)); + } + return metadata; } + } - /** - * Attempt to add table metadata to the cache - * @return The result of trying to cache the metadata - */ - private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( - TableIdentifier tableIdentifier, - TableMetadata metadata, - PolarisCallContext callContext, - PolarisEntityManager entityManager, - PolarisResolutionManifestCatalogView resolvedEntityView) { - PolarisResolvedPathWrapper resolvedEntities = - resolvedEntityView.getPassthroughResolvedPath( - tableIdentifier, PolarisEntitySubType.TABLE); - if (resolvedEntities == null) { - return new PolarisMetaStoreManager.EntityResult( - PolarisMetaStoreManager.ReturnStatus.ENTITY_NOT_FOUND, null); - } else { - TableLikeEntity tableEntity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); - TableMetadataEntity tableMetadataEntity = - new TableMetadataEntity.Builder(metadata.location(), TableMetadataParser.toJson(metadata)) - .setCatalogId(tableEntity.getCatalogId()) - .setParentId(tableEntity.getId()) - .setId(entityManager.getMetaStoreManager().generateNewEntityId(callContext).getId()) - .build(); - return entityManager - .getMetaStoreManager() - .createEntityIfNotExists( - callContext, - PolarisEntity.toCoreList(resolvedEntities.getRawFullPath()), - tableMetadataEntity); - } + /** + * Attempt to add table metadata to the cache + * + * @return The result of trying to cache the metadata + */ + private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( + TableIdentifier tableIdentifier, + TableMetadata metadata, + PolarisCallContext callContext, + PolarisEntityManager entityManager, + PolarisResolutionManifestCatalogView resolvedEntityView) { + PolarisResolvedPathWrapper resolvedEntities = + resolvedEntityView.getPassthroughResolvedPath(tableIdentifier, PolarisEntitySubType.TABLE); + if (resolvedEntities == null) { + return new PolarisMetaStoreManager.EntityResult( + PolarisMetaStoreManager.ReturnStatus.ENTITY_NOT_FOUND, null); + } else { + TableLikeEntity tableEntity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); + TableMetadataEntity tableMetadataEntity = + new TableMetadataEntity.Builder(metadata.location(), TableMetadataParser.toJson(metadata)) + .setCatalogId(tableEntity.getCatalogId()) + .setParentId(tableEntity.getId()) + .setId(entityManager.getMetaStoreManager().generateNewEntityId(callContext).getId()) + .build(); + return entityManager + .getMetaStoreManager() + .createEntityIfNotExists( + callContext, + PolarisEntity.toCoreList(resolvedEntities.getRawFullPath()), + tableMetadataEntity); } + } - /** - * Return the cached {@link Table} entity, if one exists - */ - private static @NotNull Optional loadCachedTableMetadata( - TableIdentifier tableIdentifier, - PolarisCallContext callContext, - PolarisEntityManager entityManager, - PolarisResolutionManifestCatalogView resolvedEntityView) { - PolarisResolvedPathWrapper resolvedEntities = - resolvedEntityView.getPassthroughResolvedPath( - tableIdentifier, PolarisEntitySubType.TABLE); - if (resolvedEntities == null) { - return Optional.empty(); - } else { - TableLikeEntity entity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); - String metadataLocation = entity.getMetadataLocation(); - PolarisMetaStoreManager.EntityResult metadataEntityResult = entityManager - .getMetaStoreManager() - .readEntityByName( - callContext, - PolarisEntity.toCoreList(resolvedEntities.getRawFullPath()), - PolarisEntityType.TABLE_METADATA, - PolarisEntitySubType.ANY_SUBTYPE, - metadataLocation); - return Optional - .ofNullable(metadataEntityResult.getEntity()) - .map(metadataEntity -> { - TableMetadataEntity tableMetadataEntity = (TableMetadataEntity) metadataEntity; - return TableMetadataParser.fromJson(tableMetadataEntity.getContent()); - }); - } + /** Return the cached {@link Table} entity, if one exists */ + private static @NotNull Optional loadCachedTableMetadata( + TableIdentifier tableIdentifier, + PolarisCallContext callContext, + PolarisEntityManager entityManager, + PolarisResolutionManifestCatalogView resolvedEntityView) { + PolarisResolvedPathWrapper resolvedEntities = + resolvedEntityView.getPassthroughResolvedPath(tableIdentifier, PolarisEntitySubType.TABLE); + if (resolvedEntities == null) { + return Optional.empty(); + } else { + TableLikeEntity entity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); + String metadataLocation = entity.getMetadataLocation(); + PolarisMetaStoreManager.EntityResult metadataEntityResult = + entityManager + .getMetaStoreManager() + .readEntityByName( + callContext, + PolarisEntity.toCoreList(resolvedEntities.getRawFullPath()), + PolarisEntityType.TABLE_METADATA, + PolarisEntitySubType.ANY_SUBTYPE, + metadataLocation); + return Optional.ofNullable(metadataEntityResult.getEntity()) + .map( + metadataEntity -> { + TableMetadataEntity tableMetadataEntity = (TableMetadataEntity) metadataEntity; + return TableMetadataParser.fromJson(tableMetadataEntity.getContent()); + }); } + } } From 6e23be40d114d445b0e792c9f4b29d7b1aa94002 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 6 Nov 2024 21:41:06 -0800 Subject: [PATCH 08/88] logic to purge old records --- .../core/entity/TableMetadataEntity.java | 8 ++- .../persistence/MetadataCacheManager.java | 49 ++++++++++++++----- 2 files changed, 42 insertions(+), 15 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java index ae12ded449..93a1bbdd76 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java @@ -20,9 +20,13 @@ import com.fasterxml.jackson.annotation.JsonIgnore; +/** + * A {@link PolarisEntity} for storing table metadata. This can contain the raw content + * of the `metadata.json` or more granular information + */ public class TableMetadataEntity extends PolarisEntity { - private static String CONTENT_KEY = "content"; - private static String METADATA_LOCATION_KEY = "metadata_location"; + private static final String CONTENT_KEY = "content"; + private static final String METADATA_LOCATION_KEY = "metadata_location"; public TableMetadataEntity(PolarisBaseEntity sourceEntity) { super(sourceEntity); diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index c94b80a234..a235cd05b6 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.persistence; +import java.util.Collection; import java.util.Optional; import java.util.function.Supplier; import org.apache.iceberg.Table; @@ -25,6 +26,7 @@ import org.apache.iceberg.TableMetadataParser; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; @@ -112,21 +114,42 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( } else { TableLikeEntity entity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); String metadataLocation = entity.getMetadataLocation(); - PolarisMetaStoreManager.EntityResult metadataEntityResult = - entityManager - .getMetaStoreManager() - .readEntityByName( + PolarisMetaStoreManager.ListEntitiesResult metadataResult = entityManager + .getMetaStoreManager() + .listEntities( + callContext, + PolarisEntity.toCoreList(resolvedEntities.getRawFullPath()), + PolarisEntityType.TABLE_METADATA, + PolarisEntitySubType.ANY_SUBTYPE + ); + return Optional.ofNullable(metadataResult.getEntities()) + .stream() + .flatMap(Collection::stream) + .flatMap(result -> { + PolarisMetaStoreManager.EntityResult metadataEntityResult = + entityManager.getMetaStoreManager() + .loadEntity(callContext, result.getCatalogId(), result.getId()); + return Optional.ofNullable(metadataEntityResult.getEntity()) + .map(e -> (TableMetadataEntity) e) + .stream(); + }) + .filter(metadata -> { + if (metadata.getMetadataLocation().equals(metadataLocation)) { + return true; // Keep this metadata as it's the one we're interested in + } else { + LOGGER.debug(String.format("Deleting old entry for %s", metadata.getMetadataLocation())); + entityManager.getMetaStoreManager().dropEntityIfExists( callContext, PolarisEntity.toCoreList(resolvedEntities.getRawFullPath()), - PolarisEntityType.TABLE_METADATA, - PolarisEntitySubType.ANY_SUBTYPE, - metadataLocation); - return Optional.ofNullable(metadataEntityResult.getEntity()) - .map( - metadataEntity -> { - TableMetadataEntity tableMetadataEntity = (TableMetadataEntity) metadataEntity; - return TableMetadataParser.fromJson(tableMetadataEntity.getContent()); - }); + metadata, + null, + /*purge=*/ false + ); + return false; + } + }) + .findFirst() + .map(metadataEntity -> TableMetadataParser.fromJson(metadataEntity.getContent())); } } } From 5805679240855ae2d34e32b79688c77aa3aa32fa Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 6 Nov 2024 21:41:13 -0800 Subject: [PATCH 09/88] autolint --- .../core/entity/TableMetadataEntity.java | 4 +- .../persistence/MetadataCacheManager.java | 69 ++++++++++--------- 2 files changed, 38 insertions(+), 35 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java index 93a1bbdd76..0add5a1e98 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java @@ -21,8 +21,8 @@ import com.fasterxml.jackson.annotation.JsonIgnore; /** - * A {@link PolarisEntity} for storing table metadata. This can contain the raw content - * of the `metadata.json` or more granular information + * A {@link PolarisEntity} for storing table metadata. This can contain the raw content of the + * `metadata.json` or more granular information */ public class TableMetadataEntity extends PolarisEntity { private static final String CONTENT_KEY = "content"; diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index a235cd05b6..e28d97cd62 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -26,7 +26,6 @@ import org.apache.iceberg.TableMetadataParser; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; @@ -114,40 +113,44 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( } else { TableLikeEntity entity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); String metadataLocation = entity.getMetadataLocation(); - PolarisMetaStoreManager.ListEntitiesResult metadataResult = entityManager - .getMetaStoreManager() - .listEntities( - callContext, - PolarisEntity.toCoreList(resolvedEntities.getRawFullPath()), - PolarisEntityType.TABLE_METADATA, - PolarisEntitySubType.ANY_SUBTYPE - ); - return Optional.ofNullable(metadataResult.getEntities()) - .stream() - .flatMap(Collection::stream) - .flatMap(result -> { - PolarisMetaStoreManager.EntityResult metadataEntityResult = - entityManager.getMetaStoreManager() - .loadEntity(callContext, result.getCatalogId(), result.getId()); - return Optional.ofNullable(metadataEntityResult.getEntity()) - .map(e -> (TableMetadataEntity) e) - .stream(); - }) - .filter(metadata -> { - if (metadata.getMetadataLocation().equals(metadataLocation)) { - return true; // Keep this metadata as it's the one we're interested in - } else { - LOGGER.debug(String.format("Deleting old entry for %s", metadata.getMetadataLocation())); - entityManager.getMetaStoreManager().dropEntityIfExists( + PolarisMetaStoreManager.ListEntitiesResult metadataResult = + entityManager + .getMetaStoreManager() + .listEntities( callContext, PolarisEntity.toCoreList(resolvedEntities.getRawFullPath()), - metadata, - null, - /*purge=*/ false - ); - return false; - } - }) + PolarisEntityType.TABLE_METADATA, + PolarisEntitySubType.ANY_SUBTYPE); + return Optional.ofNullable(metadataResult.getEntities()).stream() + .flatMap(Collection::stream) + .flatMap( + result -> { + PolarisMetaStoreManager.EntityResult metadataEntityResult = + entityManager + .getMetaStoreManager() + .loadEntity(callContext, result.getCatalogId(), result.getId()); + return Optional.ofNullable(metadataEntityResult.getEntity()) + .map(e -> (TableMetadataEntity) e) + .stream(); + }) + .filter( + metadata -> { + if (metadata.getMetadataLocation().equals(metadataLocation)) { + return true; // Keep this metadata as it's the one we're interested in + } else { + LOGGER.debug( + String.format("Deleting old entry for %s", metadata.getMetadataLocation())); + entityManager + .getMetaStoreManager() + .dropEntityIfExists( + callContext, + PolarisEntity.toCoreList(resolvedEntities.getRawFullPath()), + metadata, + null, + /* purge= */ false); + return false; + } + }) .findFirst() .map(metadataEntity -> TableMetadataParser.fromJson(metadataEntity.getContent())); } From f32a82c773f8e25095825259b79bb8cfa17d4901 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 6 Nov 2024 21:48:23 -0800 Subject: [PATCH 10/88] stable, disabled --- .../java/org/apache/polaris/core/PolarisConfiguration.java | 4 ++-- .../apache/polaris/service/catalog/BasePolarisCatalog.java | 2 +- .../polaris/service/persistence/MetadataCacheManager.java | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java index 6266cbcbfb..a0109a7526 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java @@ -192,9 +192,9 @@ public static Builder builder() { .defaultValue(true) .build(); - public static final PolarisConfiguration USE_METADATA_CACHE = + public static final PolarisConfiguration METADATA_CACHE_ENABLED = PolarisConfiguration.builder() - .key("USE_METADATA_CACHE") + .key("METADATA_CACHE_ENABLED") .description( "If set to true, support serving table metadata without reading the metadata.json file.") .defaultValue(false) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index c2462a11e3..2100ebfa19 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -819,7 +819,7 @@ public TableMetadata loadTableMetadata(TableIdentifier identifier) { .getPolarisCallContext() .getConfigurationStore() .getConfiguration( - callContext.getPolarisCallContext(), PolarisConfiguration.USE_METADATA_CACHE); + callContext.getPolarisCallContext(), PolarisConfiguration.METADATA_CACHE_ENABLED); if (!useMetadataCache) { return loadTableMetadata(loadTable(identifier)); } else { diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index e28d97cd62..178417d7b2 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -90,6 +90,7 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( .setCatalogId(tableEntity.getCatalogId()) .setParentId(tableEntity.getId()) .setId(entityManager.getMetaStoreManager().generateNewEntityId(callContext).getId()) + .setCreateTimestamp(System.currentTimeMillis()) .build(); return entityManager .getMetaStoreManager() From 249ecac33d191414dd9cd27e1f041920fe5c03be Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 6 Nov 2024 23:06:49 -0800 Subject: [PATCH 11/88] passing when enabled --- .../main/java/org/apache/polaris/core/PolarisConfiguration.java | 2 +- .../polaris/service/persistence/MetadataCacheManager.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java index a0109a7526..4d9e4e2927 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java @@ -197,6 +197,6 @@ public static Builder builder() { .key("METADATA_CACHE_ENABLED") .description( "If set to true, support serving table metadata without reading the metadata.json file.") - .defaultValue(false) + .defaultValue(true) .build(); } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 178417d7b2..d9c16ba3fa 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -131,7 +131,7 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( .getMetaStoreManager() .loadEntity(callContext, result.getCatalogId(), result.getId()); return Optional.ofNullable(metadataEntityResult.getEntity()) - .map(e -> (TableMetadataEntity) e) + .map(TableMetadataEntity::of) .stream(); }) .filter( From 0db85eb527a1422839d269a21c0759bac6315b53 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 7 Nov 2024 10:09:05 -0800 Subject: [PATCH 12/88] stable --- .../core/entity/TableMetadataEntity.java | 6 +- .../core/persistence/EntityCacheTest.java | 2 +- .../persistence/MetadataCacheManager.java | 6 +- .../catalog/BasePolarisCatalogTest.java | 88 ++++++++++++++++++- 4 files changed, 94 insertions(+), 8 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java index 0add5a1e98..ac9d089f9f 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java @@ -51,12 +51,9 @@ public String getMetadataLocation() { public static class Builder extends PolarisEntity.BaseBuilder { - public Builder(String metadataLocation, String content) { + public Builder() { super(); - setName(metadataLocation); setType(PolarisEntityType.TABLE_METADATA); - setMetadataLocation(metadataLocation); - setContent(content); } @Override @@ -71,6 +68,7 @@ public TableMetadataEntity.Builder setContent(String content) { public TableMetadataEntity.Builder setMetadataLocation(String metadataLocation) { this.internalProperties.put(METADATA_LOCATION_KEY, metadataLocation); + this.setName(metadataLocation); return this; } } diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java index 01062f335f..b1033022f9 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java @@ -86,7 +86,7 @@ public EntityCacheTest() { callCtx = new PolarisCallContext(metaStore, diagServices); metaStoreManager = new PolarisMetaStoreManagerImpl(); - // bootstrap the mata store with our test schema + // bootstrap the metastore with our test schema tm = new PolarisTestMetaStoreManager(metaStoreManager, callCtx); tm.testCreateTestCatalog(); } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index d9c16ba3fa..392459400e 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -86,11 +86,13 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( } else { TableLikeEntity tableEntity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); TableMetadataEntity tableMetadataEntity = - new TableMetadataEntity.Builder(metadata.location(), TableMetadataParser.toJson(metadata)) + new TableMetadataEntity.Builder() .setCatalogId(tableEntity.getCatalogId()) .setParentId(tableEntity.getId()) .setId(entityManager.getMetaStoreManager().generateNewEntityId(callContext).getId()) .setCreateTimestamp(System.currentTimeMillis()) + .setMetadataLocation(metadata.metadataFileLocation()) + .setContent(TableMetadataParser.toJson(metadata)) .build(); return entityManager .getMetaStoreManager() @@ -137,7 +139,7 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( .filter( metadata -> { if (metadata.getMetadataLocation().equals(metadataLocation)) { - return true; // Keep this metadata as it's the one we're interested in + return true; } else { LOGGER.debug( String.format("Deleting old entry for %s", metadata.getMetadataLocation())); diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java index 63f859b768..4551aea97c 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java @@ -25,8 +25,10 @@ import com.google.common.collect.ImmutableMap; import java.io.IOException; +import java.nio.file.Path; import java.time.Clock; import java.util.Arrays; +import java.util.Collections; import java.util.EnumMap; import java.util.HashMap; import java.util.List; @@ -43,6 +45,7 @@ import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.TableOperations; import org.apache.iceberg.catalog.CatalogTests; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.SupportsNamespaces; @@ -77,6 +80,7 @@ import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisMetaStoreSession; +import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView; import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisStorageIntegration; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; @@ -85,6 +89,7 @@ import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.admin.PolarisAdminService; import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; +import org.apache.polaris.service.persistence.MetadataCacheManager; import org.apache.polaris.service.task.TableCleanupTaskHandler; import org.apache.polaris.service.task.TaskExecutor; import org.apache.polaris.service.task.TaskFileIOSupplier; @@ -98,6 +103,7 @@ import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.mockito.Mockito; import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; @@ -125,6 +131,7 @@ public class BasePolarisCatalogTest extends CatalogTests { private PolarisEntityManager entityManager; private AuthenticatedPolarisPrincipal authenticatedRoot; private PolarisEntity catalogEntity; + private PolarisResolutionManifestCatalogView passthroughView; @BeforeEach @SuppressWarnings("unchecked") @@ -200,7 +207,7 @@ public void before() { .setStorageConfigurationInfo(storageConfigModel, storageLocation) .build()); - PolarisPassthroughResolutionView passthroughView = + passthroughView = new PolarisPassthroughResolutionView( callContext, entityManager, authenticatedRoot, CATALOG_NAME); TaskExecutor taskExecutor = Mockito.mock(); @@ -1384,4 +1391,83 @@ public void testFileIOWrapper() { .getFirst())); Assertions.assertThat(measured.getNumDeletedFiles()).as("A table was deleted").isGreaterThan(0); } + + private Schema buildSchema(int fields) { + Types.NestedField[] fieldsArray = new Types.NestedField[fields]; + for (int i = 0; i < fields; i++) { + fieldsArray[i] = Types.NestedField.optional(i, "field_" + i, Types.IntegerType.get()); + } + return new Schema(fieldsArray); + } + + private TableMetadata createTableMetadata(String location, Schema schema) { + return TableMetadata + .newTableMetadata( + schema, + PartitionSpec.unpartitioned(), + location, + Collections.emptyMap() + ); + } + + @Test + public void testMetadataCachingWithManualFallback() { + Namespace namespace = Namespace.of("manual-namespace"); + TableIdentifier tableIdentifier = TableIdentifier.of(namespace, "t1"); + + Schema schema = buildSchema(10); + + catalog.createNamespace(namespace); + Table createdTable = catalog.createTable(tableIdentifier, schema); + TableMetadata originalMetadata = ((BaseTable)createdTable).operations().current(); + + TableMetadata loadedMetadata = MetadataCacheManager + .loadTableMetadata( + tableIdentifier, + polarisContext, + entityManager, + passthroughView, + () -> originalMetadata); + + // The first time, the fallback is called + Assertions.assertThat(loadedMetadata).isSameAs(originalMetadata); + + TableMetadata cachedMetadata = MetadataCacheManager + .loadTableMetadata( + tableIdentifier, + polarisContext, + entityManager, + passthroughView, + () -> { + throw new IllegalStateException("Fell back even though a cache entry should exist!"); + }); + + // The second time, it's loaded from the cache + Assertions.assertThat(cachedMetadata).isNotSameAs(originalMetadata); + + // The content should match what was cached + Assertions.assertThat(TableMetadataParser.toJson(cachedMetadata)) + .isEqualTo(TableMetadataParser.toJson(loadedMetadata)); + + // Update the table + TableOperations tableOps = catalog.newTableOps(tableIdentifier); + TableMetadata updatedMetadata = + tableOps.current().updateLocation(originalMetadata.location() + "-updated"); + tableOps.commit( + tableOps.current(), + updatedMetadata); + TableMetadata spyUpdatedMetadata = Mockito.spy(updatedMetadata); + Mockito.doReturn("spy-location").when(spyUpdatedMetadata).metadataFileLocation(); + + // Read from the cache; it should detect a chance due to the update and load the new fallback + TableMetadata reloadedMetadata = MetadataCacheManager + .loadTableMetadata( + tableIdentifier, + polarisContext, + entityManager, + passthroughView, + () -> spyUpdatedMetadata); + + Assertions.assertThat(reloadedMetadata).isSameAs(spyUpdatedMetadata); + } } From 6f0d9df3231274d59a3e33b4f02a36878040ff47 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 7 Nov 2024 10:09:08 -0800 Subject: [PATCH 13/88] autolint --- .../catalog/BasePolarisCatalogTest.java | 29 +++++++------------ 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java index 4551aea97c..524f07b0a2 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java @@ -25,7 +25,6 @@ import com.google.common.collect.ImmutableMap; import java.io.IOException; -import java.nio.file.Path; import java.time.Clock; import java.util.Arrays; import java.util.Collections; @@ -103,7 +102,6 @@ import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; import org.mockito.Mockito; import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; @@ -1401,13 +1399,8 @@ private Schema buildSchema(int fields) { } private TableMetadata createTableMetadata(String location, Schema schema) { - return TableMetadata - .newTableMetadata( - schema, - PartitionSpec.unpartitioned(), - location, - Collections.emptyMap() - ); + return TableMetadata.newTableMetadata( + schema, PartitionSpec.unpartitioned(), location, Collections.emptyMap()); } @Test @@ -1419,10 +1412,10 @@ public void testMetadataCachingWithManualFallback() { catalog.createNamespace(namespace); Table createdTable = catalog.createTable(tableIdentifier, schema); - TableMetadata originalMetadata = ((BaseTable)createdTable).operations().current(); + TableMetadata originalMetadata = ((BaseTable) createdTable).operations().current(); - TableMetadata loadedMetadata = MetadataCacheManager - .loadTableMetadata( + TableMetadata loadedMetadata = + MetadataCacheManager.loadTableMetadata( tableIdentifier, polarisContext, entityManager, @@ -1432,8 +1425,8 @@ public void testMetadataCachingWithManualFallback() { // The first time, the fallback is called Assertions.assertThat(loadedMetadata).isSameAs(originalMetadata); - TableMetadata cachedMetadata = MetadataCacheManager - .loadTableMetadata( + TableMetadata cachedMetadata = + MetadataCacheManager.loadTableMetadata( tableIdentifier, polarisContext, entityManager, @@ -1453,15 +1446,13 @@ public void testMetadataCachingWithManualFallback() { TableOperations tableOps = catalog.newTableOps(tableIdentifier); TableMetadata updatedMetadata = tableOps.current().updateLocation(originalMetadata.location() + "-updated"); - tableOps.commit( - tableOps.current(), - updatedMetadata); + tableOps.commit(tableOps.current(), updatedMetadata); TableMetadata spyUpdatedMetadata = Mockito.spy(updatedMetadata); Mockito.doReturn("spy-location").when(spyUpdatedMetadata).metadataFileLocation(); // Read from the cache; it should detect a chance due to the update and load the new fallback - TableMetadata reloadedMetadata = MetadataCacheManager - .loadTableMetadata( + TableMetadata reloadedMetadata = + MetadataCacheManager.loadTableMetadata( tableIdentifier, polarisContext, entityManager, From f25091634ecf1964943d6ec4b50c4208fab41abc Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 7 Nov 2024 11:05:03 -0800 Subject: [PATCH 14/88] autolint --- .../main/java/org/apache/polaris/core/PolarisConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java index 21f574a40b..1155767486 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java @@ -197,7 +197,7 @@ public static Builder builder() { .catalogConfig("drop-with-purge.enabled") .description( "If set to true, allows tables to be dropped with the purge parameter set to true.") - .defaultValue(true) + .defaultValue(false) .build(); public static final PolarisConfiguration METADATA_CACHE_ENABLED = From dd87d423e086ff13731d449a0f5add31251cdbd5 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 7 Nov 2024 11:05:48 -0800 Subject: [PATCH 15/88] oops --- .../java/org/apache/polaris/core/PolarisConfiguration.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java index 1155767486..a067cdd9fe 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java @@ -197,7 +197,7 @@ public static Builder builder() { .catalogConfig("drop-with-purge.enabled") .description( "If set to true, allows tables to be dropped with the purge parameter set to true.") - .defaultValue(false) + .defaultValue(true) .build(); public static final PolarisConfiguration METADATA_CACHE_ENABLED = @@ -205,6 +205,6 @@ public static Builder builder() { .key("METADATA_CACHE_ENABLED") .description( "If set to true, support serving table metadata without reading the metadata.json file.") - .defaultValue(true) + .defaultValue(false) .build(); } From c46eb03e19997b37e97431fe46633cdd270c63c5 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 7 Nov 2024 12:15:44 -0800 Subject: [PATCH 16/88] add try/catch --- .../persistence/MetadataCacheManager.java | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 392459400e..642452da95 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -94,12 +94,25 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( .setMetadataLocation(metadata.metadataFileLocation()) .setContent(TableMetadataParser.toJson(metadata)) .build(); - return entityManager - .getMetaStoreManager() - .createEntityIfNotExists( - callContext, - PolarisEntity.toCoreList(resolvedEntities.getRawFullPath()), - tableMetadataEntity); + try { + return entityManager + .getMetaStoreManager() + .createEntityIfNotExists( + callContext, + PolarisEntity.toCoreList(resolvedEntities.getRawFullPath()), + tableMetadataEntity); + } catch (RuntimeException e) { + // PersistenceException (& other extension-specific exceptions) may not be in scope, + // but we can make a best-effort attempt to swallow it and just forego caching + if (e.toString().contains("PersistenceException")) { + return new PolarisMetaStoreManager.EntityResult( + PolarisMetaStoreManager.ReturnStatus.UNEXPECTED_ERROR_SIGNALED, + e.getMessage() + ); + } else { + throw e; + } + } } } From 1e1b953e1a95ced4db6492850f2c2e3713f1edc6 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 7 Nov 2024 12:15:48 -0800 Subject: [PATCH 17/88] autolint --- .../polaris/service/persistence/MetadataCacheManager.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 642452da95..4912498339 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -106,9 +106,7 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( // but we can make a best-effort attempt to swallow it and just forego caching if (e.toString().contains("PersistenceException")) { return new PolarisMetaStoreManager.EntityResult( - PolarisMetaStoreManager.ReturnStatus.UNEXPECTED_ERROR_SIGNALED, - e.getMessage() - ); + PolarisMetaStoreManager.ReturnStatus.UNEXPECTED_ERROR_SIGNALED, e.getMessage()); } else { throw e; } From a164807a04ddace6f91bb8cb683f9841d1f5d3f1 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 7 Nov 2024 20:35:59 -0800 Subject: [PATCH 18/88] added drop-on-drop --- .../PolarisMetaStoreManagerImpl.java | 7 +++++ .../service/catalog/BasePolarisCatalog.java | 8 +++++ .../persistence/MetadataCacheManager.java | 30 ++++++++++++++++++- 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index b05129fa3d..154c8dd1a4 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -24,15 +24,20 @@ import com.fasterxml.jackson.databind.ObjectMapper; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; + +import org.apache.iceberg.TableMetadata; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.entity.AsyncTaskType; import org.apache.polaris.core.entity.PolarisBaseEntity; @@ -49,6 +54,8 @@ import org.apache.polaris.core.entity.PolarisPrincipalSecrets; import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.entity.PolarisTaskConstants; +import org.apache.polaris.core.entity.TableLikeEntity; +import org.apache.polaris.core.entity.TableMetadataEntity; import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index bdec582b5e..b361dc5237 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -1890,6 +1890,14 @@ private void updateTableLike(TableIdentifier identifier, PolarisEntity entity) { } } + // Purge table metadata, if it exists: + MetadataCacheManager.dropTableMetadata( + identifier, + callContext.getPolarisCallContext(), + entityManager, + resolvedEntityView); + + // Drop the table: return getMetaStoreManager() .dropEntityIfExists( getCurrentPolarisContext(), diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 4912498339..d0ef5da619 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -120,6 +120,34 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( PolarisCallContext callContext, PolarisEntityManager entityManager, PolarisResolutionManifestCatalogView resolvedEntityView) { + return loadCachedTableMetadataImpl( + tableIdentifier, + callContext, + entityManager, + resolvedEntityView, + false); + } + + public static void dropTableMetadata( + TableIdentifier tableIdentifier, + PolarisCallContext callContext, + PolarisEntityManager entityManager, + PolarisResolutionManifestCatalogView resolvedEntityView) { + loadCachedTableMetadataImpl( + tableIdentifier, + callContext, + entityManager, + resolvedEntityView, + true); + } + + /** Return the cached {@link Table} entity, if one exists */ + private static @NotNull Optional loadCachedTableMetadataImpl( + TableIdentifier tableIdentifier, + PolarisCallContext callContext, + PolarisEntityManager entityManager, + PolarisResolutionManifestCatalogView resolvedEntityView, + boolean dropEverything) { PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getPassthroughResolvedPath(tableIdentifier, PolarisEntitySubType.TABLE); if (resolvedEntities == null) { @@ -149,7 +177,7 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( }) .filter( metadata -> { - if (metadata.getMetadataLocation().equals(metadataLocation)) { + if (metadata.getMetadataLocation().equals(metadataLocation) && !dropEverything) { return true; } else { LOGGER.debug( From 6ed3f97f406fc6eb61152fd6070679819474fc32 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 7 Nov 2024 20:36:03 -0800 Subject: [PATCH 19/88] autolint --- .../persistence/PolarisMetaStoreManagerImpl.java | 7 ------- .../polaris/service/catalog/BasePolarisCatalog.java | 5 +---- .../service/persistence/MetadataCacheManager.java | 12 ++---------- 3 files changed, 3 insertions(+), 21 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index 154c8dd1a4..b05129fa3d 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -24,20 +24,15 @@ import com.fasterxml.jackson.databind.ObjectMapper; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; - -import org.apache.iceberg.TableMetadata; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.entity.AsyncTaskType; import org.apache.polaris.core.entity.PolarisBaseEntity; @@ -54,8 +49,6 @@ import org.apache.polaris.core.entity.PolarisPrincipalSecrets; import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.entity.PolarisTaskConstants; -import org.apache.polaris.core.entity.TableLikeEntity; -import org.apache.polaris.core.entity.TableMetadataEntity; import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index b361dc5237..596838b425 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -1892,10 +1892,7 @@ private void updateTableLike(TableIdentifier identifier, PolarisEntity entity) { // Purge table metadata, if it exists: MetadataCacheManager.dropTableMetadata( - identifier, - callContext.getPolarisCallContext(), - entityManager, - resolvedEntityView); + identifier, callContext.getPolarisCallContext(), entityManager, resolvedEntityView); // Drop the table: return getMetaStoreManager() diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index d0ef5da619..c66ffac547 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -121,11 +121,7 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( PolarisEntityManager entityManager, PolarisResolutionManifestCatalogView resolvedEntityView) { return loadCachedTableMetadataImpl( - tableIdentifier, - callContext, - entityManager, - resolvedEntityView, - false); + tableIdentifier, callContext, entityManager, resolvedEntityView, false); } public static void dropTableMetadata( @@ -134,11 +130,7 @@ public static void dropTableMetadata( PolarisEntityManager entityManager, PolarisResolutionManifestCatalogView resolvedEntityView) { loadCachedTableMetadataImpl( - tableIdentifier, - callContext, - entityManager, - resolvedEntityView, - true); + tableIdentifier, callContext, entityManager, resolvedEntityView, true); } /** Return the cached {@link Table} entity, if one exists */ From 35b852f1d71416ac11e275902ce8308bfc2ef023 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 11 Nov 2024 17:30:10 -0800 Subject: [PATCH 20/88] refactor to remove new entity --- .../polaris/core/PolarisConfiguration.java | 46 ++++- .../core/entity/PolarisEntityType.java | 3 +- .../polaris/core/entity/TableLikeEntity.java | 14 ++ .../core/entity/TableMetadataEntity.java | 75 -------- .../service/catalog/BasePolarisCatalog.java | 11 +- .../persistence/MetadataCacheManager.java | 178 ++++++------------ .../catalog/BasePolarisCatalogTest.java | 15 +- 7 files changed, 130 insertions(+), 212 deletions(-) delete mode 100644 polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java diff --git a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java index a067cdd9fe..86985f44e0 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java @@ -20,6 +20,8 @@ import java.util.List; import java.util.Optional; +import java.util.function.Function; + import org.apache.polaris.core.admin.model.StorageConfigInfo; public class PolarisConfiguration { @@ -29,15 +31,23 @@ public class PolarisConfiguration { public final T defaultValue; private final Optional catalogConfigImpl; private final Class typ; + private final Optional> validation; @SuppressWarnings("unchecked") public PolarisConfiguration( - String key, String description, T defaultValue, Optional catalogConfig) { + String key, + String description, + T defaultValue, + Optional catalogConfig, + Optional> validation) { this.key = key; this.description = description; this.defaultValue = defaultValue; this.catalogConfigImpl = catalogConfig; this.typ = (Class) defaultValue.getClass(); + this.validation = validation; + + validate(cast(defaultValue)); } public boolean hasCatalogConfig() { @@ -52,7 +62,18 @@ public String catalogConfig() { } T cast(Object value) { - return this.typ.cast(value); + T result = this.typ.cast(value); + validate(result); + return result; + } + + private void validate(T value) { + this.validation.ifPresent(v -> { + if (!v.apply(value)) { + throw new IllegalArgumentException( + String.format("Configuration %s has invalid value %s", key, defaultValue)); + } + }); } public static class Builder { @@ -60,6 +81,7 @@ public static class Builder { private String description; private T defaultValue; private Optional catalogConfig = Optional.empty(); + private Optional> validation = Optional.empty(); public Builder key(String key) { this.key = key; @@ -81,11 +103,16 @@ public Builder catalogConfig(String catalogConfig) { return this; } + public Builder validation(Function validation) { + this.validation = Optional.of(validation); + return this; + } + public PolarisConfiguration build() { if (key == null || description == null || defaultValue == null) { throw new IllegalArgumentException("key, description, and defaultValue are required"); } - return new PolarisConfiguration<>(key, description, defaultValue, catalogConfig); + return new PolarisConfiguration<>(key, description, defaultValue, catalogConfig, validation); } } @@ -200,11 +227,14 @@ public static Builder builder() { .defaultValue(true) .build(); - public static final PolarisConfiguration METADATA_CACHE_ENABLED = - PolarisConfiguration.builder() - .key("METADATA_CACHE_ENABLED") + public static final Long METADATA_CACHE_MAX_BYTES_NO_CACHING = 0L; + public static final PolarisConfiguration METADATA_CACHE_MAX_BYTES = + PolarisConfiguration.builder() + .key("METADATA_CACHE_MAX_BYTES") .description( - "If set to true, support serving table metadata without reading the metadata.json file.") - .defaultValue(false) + "If nonzero, the max size a table's metadata can be in order to be cached in the persistence layer." + + " If zero, no metadata will be cached or served from the cache. If -1, all metadata will be cached.") + .defaultValue(METADATA_CACHE_MAX_BYTES_NO_CACHING) + .validation(value -> value >= -1) .build(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityType.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityType.java index 308dcb8186..fa3a2d6618 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityType.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityType.java @@ -34,8 +34,7 @@ public enum PolarisEntityType { // generic table is either a view or a real table TABLE_LIKE(7, NAMESPACE, false, false), TASK(8, ROOT, false, false), - FILE(9, TABLE_LIKE, false, false), - TABLE_METADATA(10, TABLE_LIKE, false, false); + FILE(9, TABLE_LIKE, false, false); // to efficiently map a code to its corresponding entity type, use a reverse array which // is initialized below diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/TableLikeEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/TableLikeEntity.java index 968598b93e..fb026c24cc 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/TableLikeEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/TableLikeEntity.java @@ -29,6 +29,10 @@ public class TableLikeEntity extends PolarisEntity { // of the internalProperties JSON file. public static final String METADATA_LOCATION_KEY = "metadata-location"; + // For applicable types, this key on the "internalProperties" map will return the content of the + // metadata.json file located at `METADATA_LOCATION_KEY` + private static final String METADATA_CONTENT_KEY = "metadata-content"; + public static final String USER_SPECIFIED_WRITE_DATA_LOCATION_KEY = "write.data.path"; public static final String USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY = "write.metadata.path"; @@ -67,6 +71,11 @@ public String getMetadataLocation() { return getInternalPropertiesAsMap().get(METADATA_LOCATION_KEY); } + @JsonIgnore + public String getMetadataContent() { + return getInternalPropertiesAsMap().get(METADATA_CONTENT_KEY); + } + @JsonIgnore public Optional getLastAdmittedNotificationTimestamp() { return Optional.ofNullable( @@ -121,6 +130,11 @@ public Builder setMetadataLocation(String location) { return this; } + public Builder setMetadataContent(String content) { + internalProperties.put(METADATA_CONTENT_KEY, content); + return this; + } + public Builder setLastNotificationTimestamp(long timestamp) { internalProperties.put(LAST_ADMITTED_NOTIFICATION_TIMESTAMP_KEY, String.valueOf(timestamp)); return this; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java deleted file mode 100644 index ac9d089f9f..0000000000 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/TableMetadataEntity.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.core.entity; - -import com.fasterxml.jackson.annotation.JsonIgnore; - -/** - * A {@link PolarisEntity} for storing table metadata. This can contain the raw content of the - * `metadata.json` or more granular information - */ -public class TableMetadataEntity extends PolarisEntity { - private static final String CONTENT_KEY = "content"; - private static final String METADATA_LOCATION_KEY = "metadata_location"; - - public TableMetadataEntity(PolarisBaseEntity sourceEntity) { - super(sourceEntity); - } - - public static TableMetadataEntity of(PolarisBaseEntity sourceEntity) { - if (sourceEntity != null) { - return new TableMetadataEntity(sourceEntity); - } - return null; - } - - @JsonIgnore - public String getContent() { - return getInternalPropertiesAsMap().get(CONTENT_KEY); - } - - @JsonIgnore - public String getMetadataLocation() { - return getInternalPropertiesAsMap().get(METADATA_LOCATION_KEY); - } - - public static class Builder - extends PolarisEntity.BaseBuilder { - public Builder() { - super(); - setType(PolarisEntityType.TABLE_METADATA); - } - - @Override - public TableMetadataEntity build() { - return new TableMetadataEntity(buildBase()); - } - - public TableMetadataEntity.Builder setContent(String content) { - this.internalProperties.put(CONTENT_KEY, content); - return this; - } - - public TableMetadataEntity.Builder setMetadataLocation(String metadataLocation) { - this.internalProperties.put(METADATA_LOCATION_KEY, metadataLocation); - this.setName(metadataLocation); - return this; - } - } -} diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 596838b425..acdb09bdc6 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -823,18 +823,19 @@ public Map getCredentialConfig( } public TableMetadata loadTableMetadata(TableIdentifier identifier) { - boolean useMetadataCache = + long maxMetadataCacheBytes = callContext .getPolarisCallContext() .getConfigurationStore() .getConfiguration( - callContext.getPolarisCallContext(), PolarisConfiguration.METADATA_CACHE_ENABLED); - if (!useMetadataCache) { + callContext.getPolarisCallContext(), PolarisConfiguration.METADATA_CACHE_MAX_BYTES); + if (maxMetadataCacheBytes == PolarisConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING) { return loadTableMetadata(loadTable(identifier)); } else { Supplier fallback = () -> loadTableMetadata(loadTable(identifier)); return MetadataCacheManager.loadTableMetadata( identifier, + maxMetadataCacheBytes, callContext.getPolarisCallContext(), entityManager, resolvedEntityView, @@ -1890,10 +1891,6 @@ private void updateTableLike(TableIdentifier identifier, PolarisEntity entity) { } } - // Purge table metadata, if it exists: - MetadataCacheManager.dropTableMetadata( - identifier, callContext.getPolarisCallContext(), entityManager, resolvedEntityView); - // Drop the table: return getMetaStoreManager() .dropEntityIfExists( diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index c66ffac547..510fa64d40 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -18,7 +18,7 @@ */ package org.apache.polaris.service.persistence; -import java.util.Collection; +import java.nio.charset.StandardCharsets; import java.util.Optional; import java.util.function.Supplier; import org.apache.iceberg.Table; @@ -26,40 +26,51 @@ import org.apache.iceberg.TableMetadataParser; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.PolarisConfiguration; import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntitySubType; -import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.TableLikeEntity; -import org.apache.polaris.core.entity.TableMetadataEntity; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView; -import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class MetadataCacheManager { private static final Logger LOGGER = LoggerFactory.getLogger(MetadataCacheManager.class); - /** Load the cached {@link Table} or fall back to `fallback` if one doesn't exist */ + /** + * Load the cached {@link Table} or fall back to `fallback` if one doesn't exist + * If the metadata is not currently cached, it may be added to the cache. + */ public static TableMetadata loadTableMetadata( TableIdentifier tableIdentifier, + long maxBytesToCache, PolarisCallContext callContext, PolarisEntityManager entityManager, PolarisResolutionManifestCatalogView resolvedEntityView, Supplier fallback) { LOGGER.debug(String.format("Loading cached metadata for %s", tableIdentifier)); - Optional cachedMetadata = - loadCachedTableMetadata(tableIdentifier, callContext, entityManager, resolvedEntityView); - if (cachedMetadata.isPresent()) { + PolarisResolvedPathWrapper resolvedEntities = + resolvedEntityView.getResolvedPath(tableIdentifier, PolarisEntitySubType.TABLE); + TableLikeEntity tableLikeEntity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); + Optional cachedMetadata = Optional + .ofNullable(tableLikeEntity) + .map(TableLikeEntity::getMetadataContent) + .map(TableMetadataParser::fromJson); + boolean isCacheValid = cachedMetadata + .map(TableMetadata::metadataFileLocation) + .map(s -> s.equals(tableLikeEntity.getMetadataLocation())) + .orElse(false); + if (isCacheValid) { LOGGER.debug(String.format("Using cached metadata for %s", tableIdentifier)); return cachedMetadata.get(); } else { TableMetadata metadata = fallback.get(); PolarisMetaStoreManager.EntityResult cacheResult = cacheTableMetadata( - tableIdentifier, metadata, callContext, entityManager, resolvedEntityView); + tableLikeEntity, metadata, maxBytesToCache, callContext, entityManager, resolvedEntityView); if (!cacheResult.isSuccess()) { LOGGER.debug(String.format("Failed to cache metadata for %s", tableIdentifier)); } @@ -73,120 +84,55 @@ public static TableMetadata loadTableMetadata( * @return The result of trying to cache the metadata */ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( - TableIdentifier tableIdentifier, + TableLikeEntity tableLikeEntity, TableMetadata metadata, + long maxBytesToCache, PolarisCallContext callContext, PolarisEntityManager entityManager, PolarisResolutionManifestCatalogView resolvedEntityView) { - PolarisResolvedPathWrapper resolvedEntities = - resolvedEntityView.getPassthroughResolvedPath(tableIdentifier, PolarisEntitySubType.TABLE); - if (resolvedEntities == null) { - return new PolarisMetaStoreManager.EntityResult( - PolarisMetaStoreManager.ReturnStatus.ENTITY_NOT_FOUND, null); - } else { - TableLikeEntity tableEntity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); - TableMetadataEntity tableMetadataEntity = - new TableMetadataEntity.Builder() - .setCatalogId(tableEntity.getCatalogId()) - .setParentId(tableEntity.getId()) - .setId(entityManager.getMetaStoreManager().generateNewEntityId(callContext).getId()) - .setCreateTimestamp(System.currentTimeMillis()) - .setMetadataLocation(metadata.metadataFileLocation()) - .setContent(TableMetadataParser.toJson(metadata)) - .build(); - try { - return entityManager - .getMetaStoreManager() - .createEntityIfNotExists( - callContext, - PolarisEntity.toCoreList(resolvedEntities.getRawFullPath()), - tableMetadataEntity); - } catch (RuntimeException e) { - // PersistenceException (& other extension-specific exceptions) may not be in scope, - // but we can make a best-effort attempt to swallow it and just forego caching - if (e.toString().contains("PersistenceException")) { - return new PolarisMetaStoreManager.EntityResult( - PolarisMetaStoreManager.ReturnStatus.UNEXPECTED_ERROR_SIGNALED, e.getMessage()); - } else { - throw e; + String json = TableMetadataParser.toJson(metadata); + // We should not reach this method in this case, but check just in case... + if (maxBytesToCache != PolarisConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING) { + long sizeInBytes = json.getBytes(StandardCharsets.UTF_8).length; + if (sizeInBytes > maxBytesToCache) { + LOGGER.debug(String.format( + "Will not cache metadata for %s; metadata is %d bytes and the limit is %d", + tableLikeEntity.getTableIdentifier(), + sizeInBytes, + maxBytesToCache + )); + return new PolarisMetaStoreManager.EntityResult(PolarisMetaStoreManager.ReturnStatus.SUCCESS, null); + } else { + LOGGER.debug(String.format("Caching metadata for %s", tableLikeEntity.getTableIdentifier())); + TableLikeEntity newTableLikeEntity = new TableLikeEntity.Builder(tableLikeEntity) + .setMetadataContent(json) + .build(); + PolarisResolvedPathWrapper resolvedPath = + resolvedEntityView.getResolvedPath(tableLikeEntity.getTableIdentifier(), PolarisEntitySubType.TABLE); + try { + return entityManager.getMetaStoreManager().updateEntityPropertiesIfNotChanged( + callContext, + PolarisEntity.toCoreList(resolvedPath.getRawFullPath()), + newTableLikeEntity); + } catch (RuntimeException e) { + // PersistenceException (& other extension-specific exceptions) may not be in scope, + // but we can make a best-effort attempt to swallow it and just forego caching + if (e.toString().contains("PersistenceException")) { + LOGGER.debug(String.format( + "Encountered an error while caching %s: %s", tableLikeEntity.getTableIdentifier(), e)); + return new PolarisMetaStoreManager.EntityResult( + PolarisMetaStoreManager.ReturnStatus.UNEXPECTED_ERROR_SIGNALED, e.getMessage()); + } else { + throw e; + } } } - } - } - - /** Return the cached {@link Table} entity, if one exists */ - private static @NotNull Optional loadCachedTableMetadata( - TableIdentifier tableIdentifier, - PolarisCallContext callContext, - PolarisEntityManager entityManager, - PolarisResolutionManifestCatalogView resolvedEntityView) { - return loadCachedTableMetadataImpl( - tableIdentifier, callContext, entityManager, resolvedEntityView, false); - } - - public static void dropTableMetadata( - TableIdentifier tableIdentifier, - PolarisCallContext callContext, - PolarisEntityManager entityManager, - PolarisResolutionManifestCatalogView resolvedEntityView) { - loadCachedTableMetadataImpl( - tableIdentifier, callContext, entityManager, resolvedEntityView, true); - } - - /** Return the cached {@link Table} entity, if one exists */ - private static @NotNull Optional loadCachedTableMetadataImpl( - TableIdentifier tableIdentifier, - PolarisCallContext callContext, - PolarisEntityManager entityManager, - PolarisResolutionManifestCatalogView resolvedEntityView, - boolean dropEverything) { - PolarisResolvedPathWrapper resolvedEntities = - resolvedEntityView.getPassthroughResolvedPath(tableIdentifier, PolarisEntitySubType.TABLE); - if (resolvedEntities == null) { - return Optional.empty(); } else { - TableLikeEntity entity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); - String metadataLocation = entity.getMetadataLocation(); - PolarisMetaStoreManager.ListEntitiesResult metadataResult = - entityManager - .getMetaStoreManager() - .listEntities( - callContext, - PolarisEntity.toCoreList(resolvedEntities.getRawFullPath()), - PolarisEntityType.TABLE_METADATA, - PolarisEntitySubType.ANY_SUBTYPE); - return Optional.ofNullable(metadataResult.getEntities()).stream() - .flatMap(Collection::stream) - .flatMap( - result -> { - PolarisMetaStoreManager.EntityResult metadataEntityResult = - entityManager - .getMetaStoreManager() - .loadEntity(callContext, result.getCatalogId(), result.getId()); - return Optional.ofNullable(metadataEntityResult.getEntity()) - .map(TableMetadataEntity::of) - .stream(); - }) - .filter( - metadata -> { - if (metadata.getMetadataLocation().equals(metadataLocation) && !dropEverything) { - return true; - } else { - LOGGER.debug( - String.format("Deleting old entry for %s", metadata.getMetadataLocation())); - entityManager - .getMetaStoreManager() - .dropEntityIfExists( - callContext, - PolarisEntity.toCoreList(resolvedEntities.getRawFullPath()), - metadata, - null, - /* purge= */ false); - return false; - } - }) - .findFirst() - .map(metadataEntity -> TableMetadataParser.fromJson(metadataEntity.getContent())); + LOGGER.debug(String.format( + "Will not cache metadata for %s; metadata caching is disabled", + tableLikeEntity.getTableIdentifier() + )); + return new PolarisMetaStoreManager.EntityResult(PolarisMetaStoreManager.ReturnStatus.SUCCESS, null); } } } diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java index e9daa99eb8..0502eea281 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java @@ -35,6 +35,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; import org.apache.commons.lang3.NotImplementedException; import org.apache.iceberg.BaseTable; @@ -1555,6 +1556,7 @@ public void testMetadataCachingWithManualFallback() { TableMetadata loadedMetadata = MetadataCacheManager.loadTableMetadata( tableIdentifier, + Long.MAX_VALUE, polarisContext, entityManager, passthroughView, @@ -1566,6 +1568,7 @@ public void testMetadataCachingWithManualFallback() { TableMetadata cachedMetadata = MetadataCacheManager.loadTableMetadata( tableIdentifier, + Long.MAX_VALUE, polarisContext, entityManager, passthroughView, @@ -1585,18 +1588,22 @@ public void testMetadataCachingWithManualFallback() { TableMetadata updatedMetadata = tableOps.current().updateLocation(originalMetadata.location() + "-updated"); tableOps.commit(tableOps.current(), updatedMetadata); - TableMetadata spyUpdatedMetadata = Mockito.spy(updatedMetadata); - Mockito.doReturn("spy-location").when(spyUpdatedMetadata).metadataFileLocation(); + AtomicBoolean wasFallbackCalledAgain = new AtomicBoolean(false); // Read from the cache; it should detect a chance due to the update and load the new fallback TableMetadata reloadedMetadata = MetadataCacheManager.loadTableMetadata( tableIdentifier, + Long.MAX_VALUE, polarisContext, entityManager, passthroughView, - () -> spyUpdatedMetadata); + () -> { + wasFallbackCalledAgain.set(true); + return updatedMetadata; + }); - Assertions.assertThat(reloadedMetadata).isSameAs(spyUpdatedMetadata); + Assertions.assertThat(reloadedMetadata).isNotSameAs(cachedMetadata); + Assertions.assertThat(wasFallbackCalledAgain.get()).isTrue(); } } From 0b3df37c122bf6d9c2d76251d62e435d952d8058 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 11 Nov 2024 17:30:21 -0800 Subject: [PATCH 21/88] autolint --- .../polaris/core/PolarisConfiguration.java | 18 ++--- .../persistence/MetadataCacheManager.java | 79 +++++++++++-------- 2 files changed, 54 insertions(+), 43 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java index 86985f44e0..8a67132212 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java @@ -21,7 +21,6 @@ import java.util.List; import java.util.Optional; import java.util.function.Function; - import org.apache.polaris.core.admin.model.StorageConfigInfo; public class PolarisConfiguration { @@ -68,12 +67,13 @@ T cast(Object value) { } private void validate(T value) { - this.validation.ifPresent(v -> { - if (!v.apply(value)) { - throw new IllegalArgumentException( - String.format("Configuration %s has invalid value %s", key, defaultValue)); - } - }); + this.validation.ifPresent( + v -> { + if (!v.apply(value)) { + throw new IllegalArgumentException( + String.format("Configuration %s has invalid value %s", key, defaultValue)); + } + }); } public static class Builder { @@ -232,8 +232,8 @@ public static Builder builder() { PolarisConfiguration.builder() .key("METADATA_CACHE_MAX_BYTES") .description( - "If nonzero, the max size a table's metadata can be in order to be cached in the persistence layer." + - " If zero, no metadata will be cached or served from the cache. If -1, all metadata will be cached.") + "If nonzero, the max size a table's metadata can be in order to be cached in the persistence layer." + + " If zero, no metadata will be cached or served from the cache. If -1, all metadata will be cached.") .defaultValue(METADATA_CACHE_MAX_BYTES_NO_CACHING) .validation(value -> value >= -1) .build(); diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 510fa64d40..d7fbb19365 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -41,8 +41,8 @@ public class MetadataCacheManager { private static final Logger LOGGER = LoggerFactory.getLogger(MetadataCacheManager.class); /** - * Load the cached {@link Table} or fall back to `fallback` if one doesn't exist - * If the metadata is not currently cached, it may be added to the cache. + * Load the cached {@link Table} or fall back to `fallback` if one doesn't exist If the metadata + * is not currently cached, it may be added to the cache. */ public static TableMetadata loadTableMetadata( TableIdentifier tableIdentifier, @@ -55,14 +55,15 @@ public static TableMetadata loadTableMetadata( PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getResolvedPath(tableIdentifier, PolarisEntitySubType.TABLE); TableLikeEntity tableLikeEntity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); - Optional cachedMetadata = Optional - .ofNullable(tableLikeEntity) - .map(TableLikeEntity::getMetadataContent) - .map(TableMetadataParser::fromJson); - boolean isCacheValid = cachedMetadata - .map(TableMetadata::metadataFileLocation) - .map(s -> s.equals(tableLikeEntity.getMetadataLocation())) - .orElse(false); + Optional cachedMetadata = + Optional.ofNullable(tableLikeEntity) + .map(TableLikeEntity::getMetadataContent) + .map(TableMetadataParser::fromJson); + boolean isCacheValid = + cachedMetadata + .map(TableMetadata::metadataFileLocation) + .map(s -> s.equals(tableLikeEntity.getMetadataLocation())) + .orElse(false); if (isCacheValid) { LOGGER.debug(String.format("Using cached metadata for %s", tableIdentifier)); return cachedMetadata.get(); @@ -70,7 +71,12 @@ public static TableMetadata loadTableMetadata( TableMetadata metadata = fallback.get(); PolarisMetaStoreManager.EntityResult cacheResult = cacheTableMetadata( - tableLikeEntity, metadata, maxBytesToCache, callContext, entityManager, resolvedEntityView); + tableLikeEntity, + metadata, + maxBytesToCache, + callContext, + entityManager, + resolvedEntityView); if (!cacheResult.isSuccess()) { LOGGER.debug(String.format("Failed to cache metadata for %s", tableIdentifier)); } @@ -95,31 +101,35 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( if (maxBytesToCache != PolarisConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING) { long sizeInBytes = json.getBytes(StandardCharsets.UTF_8).length; if (sizeInBytes > maxBytesToCache) { - LOGGER.debug(String.format( - "Will not cache metadata for %s; metadata is %d bytes and the limit is %d", - tableLikeEntity.getTableIdentifier(), - sizeInBytes, - maxBytesToCache - )); - return new PolarisMetaStoreManager.EntityResult(PolarisMetaStoreManager.ReturnStatus.SUCCESS, null); + LOGGER.debug( + String.format( + "Will not cache metadata for %s; metadata is %d bytes and the limit is %d", + tableLikeEntity.getTableIdentifier(), sizeInBytes, maxBytesToCache)); + return new PolarisMetaStoreManager.EntityResult( + PolarisMetaStoreManager.ReturnStatus.SUCCESS, null); } else { - LOGGER.debug(String.format("Caching metadata for %s", tableLikeEntity.getTableIdentifier())); - TableLikeEntity newTableLikeEntity = new TableLikeEntity.Builder(tableLikeEntity) - .setMetadataContent(json) - .build(); + LOGGER.debug( + String.format("Caching metadata for %s", tableLikeEntity.getTableIdentifier())); + TableLikeEntity newTableLikeEntity = + new TableLikeEntity.Builder(tableLikeEntity).setMetadataContent(json).build(); PolarisResolvedPathWrapper resolvedPath = - resolvedEntityView.getResolvedPath(tableLikeEntity.getTableIdentifier(), PolarisEntitySubType.TABLE); + resolvedEntityView.getResolvedPath( + tableLikeEntity.getTableIdentifier(), PolarisEntitySubType.TABLE); try { - return entityManager.getMetaStoreManager().updateEntityPropertiesIfNotChanged( - callContext, - PolarisEntity.toCoreList(resolvedPath.getRawFullPath()), - newTableLikeEntity); + return entityManager + .getMetaStoreManager() + .updateEntityPropertiesIfNotChanged( + callContext, + PolarisEntity.toCoreList(resolvedPath.getRawFullPath()), + newTableLikeEntity); } catch (RuntimeException e) { // PersistenceException (& other extension-specific exceptions) may not be in scope, // but we can make a best-effort attempt to swallow it and just forego caching if (e.toString().contains("PersistenceException")) { - LOGGER.debug(String.format( - "Encountered an error while caching %s: %s", tableLikeEntity.getTableIdentifier(), e)); + LOGGER.debug( + String.format( + "Encountered an error while caching %s: %s", + tableLikeEntity.getTableIdentifier(), e)); return new PolarisMetaStoreManager.EntityResult( PolarisMetaStoreManager.ReturnStatus.UNEXPECTED_ERROR_SIGNALED, e.getMessage()); } else { @@ -128,11 +138,12 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( } } } else { - LOGGER.debug(String.format( - "Will not cache metadata for %s; metadata caching is disabled", - tableLikeEntity.getTableIdentifier() - )); - return new PolarisMetaStoreManager.EntityResult(PolarisMetaStoreManager.ReturnStatus.SUCCESS, null); + LOGGER.debug( + String.format( + "Will not cache metadata for %s; metadata caching is disabled", + tableLikeEntity.getTableIdentifier())); + return new PolarisMetaStoreManager.EntityResult( + PolarisMetaStoreManager.ReturnStatus.SUCCESS, null); } } } From 5f7c74f3081a226a1eff7518260e64f7822b21ac Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 11 Nov 2024 19:04:34 -0800 Subject: [PATCH 22/88] fixes --- .../service/persistence/MetadataCacheManager.java | 11 +++-------- .../service/catalog/BasePolarisCatalogTest.java | 9 +++------ 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index d7fbb19365..39f96a3694 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -53,18 +53,13 @@ public static TableMetadata loadTableMetadata( Supplier fallback) { LOGGER.debug(String.format("Loading cached metadata for %s", tableIdentifier)); PolarisResolvedPathWrapper resolvedEntities = - resolvedEntityView.getResolvedPath(tableIdentifier, PolarisEntitySubType.TABLE); + resolvedEntityView.getPassthroughResolvedPath(tableIdentifier, PolarisEntitySubType.TABLE); TableLikeEntity tableLikeEntity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); Optional cachedMetadata = Optional.ofNullable(tableLikeEntity) .map(TableLikeEntity::getMetadataContent) .map(TableMetadataParser::fromJson); - boolean isCacheValid = - cachedMetadata - .map(TableMetadata::metadataFileLocation) - .map(s -> s.equals(tableLikeEntity.getMetadataLocation())) - .orElse(false); - if (isCacheValid) { + if (cachedMetadata.isPresent()) { LOGGER.debug(String.format("Using cached metadata for %s", tableIdentifier)); return cachedMetadata.get(); } else { @@ -120,7 +115,7 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( .getMetaStoreManager() .updateEntityPropertiesIfNotChanged( callContext, - PolarisEntity.toCoreList(resolvedPath.getRawFullPath()), + PolarisEntity.toCoreList(resolvedPath.getRawParentPath()), newTableLikeEntity); } catch (RuntimeException e) { // PersistenceException (& other extension-specific exceptions) may not be in scope, diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java index 0502eea281..9419f07ea2 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java @@ -100,6 +100,7 @@ import org.apache.polaris.service.types.NotificationRequest; import org.apache.polaris.service.types.NotificationType; import org.apache.polaris.service.types.TableUpdateNotification; +import org.apache.spark.Partition; import org.assertj.core.api.AbstractBooleanAssert; import org.assertj.core.api.Assertions; import org.jetbrains.annotations.Nullable; @@ -1537,11 +1538,6 @@ private Schema buildSchema(int fields) { return new Schema(fieldsArray); } - private TableMetadata createTableMetadata(String location, Schema schema) { - return TableMetadata.newTableMetadata( - schema, PartitionSpec.unpartitioned(), location, Collections.emptyMap()); - } - @Test public void testMetadataCachingWithManualFallback() { Namespace namespace = Namespace.of("manual-namespace"); @@ -1586,7 +1582,7 @@ public void testMetadataCachingWithManualFallback() { // Update the table TableOperations tableOps = catalog.newTableOps(tableIdentifier); TableMetadata updatedMetadata = - tableOps.current().updateLocation(originalMetadata.location() + "-updated"); + tableOps.current().updateSchema(buildSchema(100), 100); tableOps.commit(tableOps.current(), updatedMetadata); AtomicBoolean wasFallbackCalledAgain = new AtomicBoolean(false); @@ -1604,6 +1600,7 @@ public void testMetadataCachingWithManualFallback() { }); Assertions.assertThat(reloadedMetadata).isNotSameAs(cachedMetadata); + Assertions.assertThat(reloadedMetadata.schema().columns().size()).isEqualTo(100); Assertions.assertThat(wasFallbackCalledAgain.get()).isTrue(); } } From d99a7965e07a7ea298a05ca928efa18239695e2a Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 11 Nov 2024 19:04:39 -0800 Subject: [PATCH 23/88] autolint --- .../polaris/service/catalog/BasePolarisCatalogTest.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java index 9419f07ea2..af038758c1 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java @@ -28,7 +28,6 @@ import java.io.IOException; import java.time.Clock; import java.util.Arrays; -import java.util.Collections; import java.util.EnumMap; import java.util.HashMap; import java.util.List; @@ -100,7 +99,6 @@ import org.apache.polaris.service.types.NotificationRequest; import org.apache.polaris.service.types.NotificationType; import org.apache.polaris.service.types.TableUpdateNotification; -import org.apache.spark.Partition; import org.assertj.core.api.AbstractBooleanAssert; import org.assertj.core.api.Assertions; import org.jetbrains.annotations.Nullable; @@ -1581,8 +1579,7 @@ public void testMetadataCachingWithManualFallback() { // Update the table TableOperations tableOps = catalog.newTableOps(tableIdentifier); - TableMetadata updatedMetadata = - tableOps.current().updateSchema(buildSchema(100), 100); + TableMetadata updatedMetadata = tableOps.current().updateSchema(buildSchema(100), 100); tableOps.commit(tableOps.current(), updatedMetadata); AtomicBoolean wasFallbackCalledAgain = new AtomicBoolean(false); From f1023eca8e384aeca4e4d10d24bdd68b9ddd112f Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 11 Nov 2024 19:10:20 -0800 Subject: [PATCH 24/88] stable --- .../polaris/core/entity/TableLikeEntity.java | 22 ++++++++++++++----- .../persistence/MetadataCacheManager.java | 16 +++++++------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/TableLikeEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/TableLikeEntity.java index fb026c24cc..1d6b67dc12 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/TableLikeEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/TableLikeEntity.java @@ -30,8 +30,12 @@ public class TableLikeEntity extends PolarisEntity { public static final String METADATA_LOCATION_KEY = "metadata-location"; // For applicable types, this key on the "internalProperties" map will return the content of the - // metadata.json file located at `METADATA_LOCATION_KEY` - private static final String METADATA_CONTENT_KEY = "metadata-content"; + // metadata.json file located at `METADATA_CACHE_LOCATION_KEY` + private static final String METADATA_CACHE_CONTENT_KEY = "metadata-cache-content"; + + // For applicable types, this key on the "internalProperties" map will return the location of the + // `metadata.json` that is cached in METADATA_CACHE_CONTENT_KEY + private static final String METADATA_CACHE_LOCATION_KEY = "metadata-cache-location"; public static final String USER_SPECIFIED_WRITE_DATA_LOCATION_KEY = "write.data.path"; public static final String USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY = "write.metadata.path"; @@ -72,8 +76,13 @@ public String getMetadataLocation() { } @JsonIgnore - public String getMetadataContent() { - return getInternalPropertiesAsMap().get(METADATA_CONTENT_KEY); + public String getMetadataCacheContent() { + return getInternalPropertiesAsMap().get(METADATA_CACHE_CONTENT_KEY); + } + + @JsonIgnore + public String getMetadataCacheLocationKey() { + return getInternalPropertiesAsMap().get(METADATA_CACHE_LOCATION_KEY); } @JsonIgnore @@ -130,8 +139,9 @@ public Builder setMetadataLocation(String location) { return this; } - public Builder setMetadataContent(String content) { - internalProperties.put(METADATA_CONTENT_KEY, content); + public Builder setMetadataContent(String location, String content) { + internalProperties.put(METADATA_CACHE_LOCATION_KEY, location); + internalProperties.put(METADATA_CACHE_CONTENT_KEY, content); return this; } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 39f96a3694..9171accf6f 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -55,13 +55,12 @@ public static TableMetadata loadTableMetadata( PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getPassthroughResolvedPath(tableIdentifier, PolarisEntitySubType.TABLE); TableLikeEntity tableLikeEntity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); - Optional cachedMetadata = - Optional.ofNullable(tableLikeEntity) - .map(TableLikeEntity::getMetadataContent) - .map(TableMetadataParser::fromJson); - if (cachedMetadata.isPresent()) { + boolean isCacheValid = tableLikeEntity + .getMetadataLocation() + .equals(tableLikeEntity.getMetadataCacheLocationKey()); + if (isCacheValid) { LOGGER.debug(String.format("Using cached metadata for %s", tableIdentifier)); - return cachedMetadata.get(); + return TableMetadataParser.fromJson(tableLikeEntity.getMetadataCacheContent()); } else { TableMetadata metadata = fallback.get(); PolarisMetaStoreManager.EntityResult cacheResult = @@ -105,8 +104,9 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( } else { LOGGER.debug( String.format("Caching metadata for %s", tableLikeEntity.getTableIdentifier())); - TableLikeEntity newTableLikeEntity = - new TableLikeEntity.Builder(tableLikeEntity).setMetadataContent(json).build(); + TableLikeEntity newTableLikeEntity = new TableLikeEntity.Builder(tableLikeEntity) + .setMetadataContent(tableLikeEntity.getMetadataLocation(), json) + .build(); PolarisResolvedPathWrapper resolvedPath = resolvedEntityView.getResolvedPath( tableLikeEntity.getTableIdentifier(), PolarisEntitySubType.TABLE); From 045d93db9ff345f445537537aeac2059151cec83 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 11 Nov 2024 19:10:24 -0800 Subject: [PATCH 25/88] autolint --- .../service/persistence/MetadataCacheManager.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 9171accf6f..7a22400e41 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -19,7 +19,6 @@ package org.apache.polaris.service.persistence; import java.nio.charset.StandardCharsets; -import java.util.Optional; import java.util.function.Supplier; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; @@ -55,9 +54,8 @@ public static TableMetadata loadTableMetadata( PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getPassthroughResolvedPath(tableIdentifier, PolarisEntitySubType.TABLE); TableLikeEntity tableLikeEntity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); - boolean isCacheValid = tableLikeEntity - .getMetadataLocation() - .equals(tableLikeEntity.getMetadataCacheLocationKey()); + boolean isCacheValid = + tableLikeEntity.getMetadataLocation().equals(tableLikeEntity.getMetadataCacheLocationKey()); if (isCacheValid) { LOGGER.debug(String.format("Using cached metadata for %s", tableIdentifier)); return TableMetadataParser.fromJson(tableLikeEntity.getMetadataCacheContent()); @@ -104,9 +102,10 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( } else { LOGGER.debug( String.format("Caching metadata for %s", tableLikeEntity.getTableIdentifier())); - TableLikeEntity newTableLikeEntity = new TableLikeEntity.Builder(tableLikeEntity) - .setMetadataContent(tableLikeEntity.getMetadataLocation(), json) - .build(); + TableLikeEntity newTableLikeEntity = + new TableLikeEntity.Builder(tableLikeEntity) + .setMetadataContent(tableLikeEntity.getMetadataLocation(), json) + .build(); PolarisResolvedPathWrapper resolvedPath = resolvedEntityView.getResolvedPath( tableLikeEntity.getTableIdentifier(), PolarisEntitySubType.TABLE); From 20edca50dc6898bb130faac45b7ce1bcf17fdb52 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 11 Nov 2024 19:14:34 -0800 Subject: [PATCH 26/88] polish --- .../polaris/service/persistence/MetadataCacheManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 7a22400e41..87ad914366 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -40,7 +40,7 @@ public class MetadataCacheManager { private static final Logger LOGGER = LoggerFactory.getLogger(MetadataCacheManager.class); /** - * Load the cached {@link Table} or fall back to `fallback` if one doesn't exist If the metadata + * Load the cached {@link Table} or fall back to `fallback` if one doesn't exist. If the metadata * is not currently cached, it may be added to the cache. */ public static TableMetadata loadTableMetadata( @@ -52,7 +52,7 @@ public static TableMetadata loadTableMetadata( Supplier fallback) { LOGGER.debug(String.format("Loading cached metadata for %s", tableIdentifier)); PolarisResolvedPathWrapper resolvedEntities = - resolvedEntityView.getPassthroughResolvedPath(tableIdentifier, PolarisEntitySubType.TABLE); + resolvedEntityView.getResolvedPath(tableIdentifier, PolarisEntitySubType.TABLE); TableLikeEntity tableLikeEntity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); boolean isCacheValid = tableLikeEntity.getMetadataLocation().equals(tableLikeEntity.getMetadataCacheLocationKey()); From f189e986c0fe8765b1b5c176e97deffd6cd37b0e Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 11 Nov 2024 22:08:05 -0800 Subject: [PATCH 27/88] one fix --- .../polaris/service/catalog/BasePolarisCatalog.java | 2 +- .../service/persistence/MetadataCacheManager.java | 9 ++++----- .../polaris/service/catalog/BasePolarisCatalogTest.java | 6 +++--- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index acdb09bdc6..2fc6843045 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -837,7 +837,7 @@ public TableMetadata loadTableMetadata(TableIdentifier identifier) { identifier, maxMetadataCacheBytes, callContext.getPolarisCallContext(), - entityManager, + entityManager.getMetaStoreManager(), resolvedEntityView, fallback); } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 87ad914366..b4af2a8675 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -47,7 +47,7 @@ public static TableMetadata loadTableMetadata( TableIdentifier tableIdentifier, long maxBytesToCache, PolarisCallContext callContext, - PolarisEntityManager entityManager, + PolarisMetaStoreManager metastoreManager, PolarisResolutionManifestCatalogView resolvedEntityView, Supplier fallback) { LOGGER.debug(String.format("Loading cached metadata for %s", tableIdentifier)); @@ -67,7 +67,7 @@ public static TableMetadata loadTableMetadata( metadata, maxBytesToCache, callContext, - entityManager, + metastoreManager, resolvedEntityView); if (!cacheResult.isSuccess()) { LOGGER.debug(String.format("Failed to cache metadata for %s", tableIdentifier)); @@ -86,7 +86,7 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( TableMetadata metadata, long maxBytesToCache, PolarisCallContext callContext, - PolarisEntityManager entityManager, + PolarisMetaStoreManager metaStoreManager, PolarisResolutionManifestCatalogView resolvedEntityView) { String json = TableMetadataParser.toJson(metadata); // We should not reach this method in this case, but check just in case... @@ -110,8 +110,7 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( resolvedEntityView.getResolvedPath( tableLikeEntity.getTableIdentifier(), PolarisEntitySubType.TABLE); try { - return entityManager - .getMetaStoreManager() + return metaStoreManager .updateEntityPropertiesIfNotChanged( callContext, PolarisEntity.toCoreList(resolvedPath.getRawParentPath()), diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java index af038758c1..5f9b272195 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java @@ -1552,7 +1552,7 @@ public void testMetadataCachingWithManualFallback() { tableIdentifier, Long.MAX_VALUE, polarisContext, - entityManager, + metaStoreManager, passthroughView, () -> originalMetadata); @@ -1564,7 +1564,7 @@ public void testMetadataCachingWithManualFallback() { tableIdentifier, Long.MAX_VALUE, polarisContext, - entityManager, + metaStoreManager, passthroughView, () -> { throw new IllegalStateException("Fell back even though a cache entry should exist!"); @@ -1589,7 +1589,7 @@ public void testMetadataCachingWithManualFallback() { tableIdentifier, Long.MAX_VALUE, polarisContext, - entityManager, + metaStoreManager, passthroughView, () -> { wasFallbackCalledAgain.set(true); From 5e90ba229112ae17ee66e6efaf4eebbf0cff74ef Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 11 Nov 2024 22:08:09 -0800 Subject: [PATCH 28/88] autolint --- .../service/persistence/MetadataCacheManager.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index b4af2a8675..f9712a70b3 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -29,7 +29,6 @@ import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.TableLikeEntity; -import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView; @@ -110,11 +109,10 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( resolvedEntityView.getResolvedPath( tableLikeEntity.getTableIdentifier(), PolarisEntitySubType.TABLE); try { - return metaStoreManager - .updateEntityPropertiesIfNotChanged( - callContext, - PolarisEntity.toCoreList(resolvedPath.getRawParentPath()), - newTableLikeEntity); + return metaStoreManager.updateEntityPropertiesIfNotChanged( + callContext, + PolarisEntity.toCoreList(resolvedPath.getRawParentPath()), + newTableLikeEntity); } catch (RuntimeException e) { // PersistenceException (& other extension-specific exceptions) may not be in scope, // but we can make a best-effort attempt to swallow it and just forego caching From fbdce6ae86f2fd65d9d5f98921f39e2acca99b0b Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 11 Nov 2024 22:12:59 -0800 Subject: [PATCH 29/88] ? --- .../org/apache/polaris/service/catalog/BasePolarisCatalog.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 2fc6843045..8fb2e35f8f 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -837,7 +837,7 @@ public TableMetadata loadTableMetadata(TableIdentifier identifier) { identifier, maxMetadataCacheBytes, callContext.getPolarisCallContext(), - entityManager.getMetaStoreManager(), + metaStoreManager, resolvedEntityView, fallback); } From f1461e546713d8edb81bd952846453f4de7ed097 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 13 Nov 2024 22:24:17 -0800 Subject: [PATCH 30/88] more widespread use of loadTableMetadata --- .../catalog/PolarisCatalogHandlerWrapper.java | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java index ef2dff86f1..443c894f98 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java @@ -612,7 +612,14 @@ public LoadTableResponse createTableDirectWithWriteDelegation( .create(); if (table instanceof BaseTable baseTable) { - TableMetadata tableMetadata = baseTable.operations().current(); + + final TableMetadata tableMetadata; + if (baseCatalog instanceof BasePolarisCatalog basePolarisCatalog) { + tableMetadata = basePolarisCatalog.loadTableMetadata(tableIdentifier); + } else { + tableMetadata = baseTable.operations().current(); + } + LoadTableResponse.Builder responseBuilder = LoadTableResponse.builder().withTableMetadata(tableMetadata); if (baseCatalog instanceof SupportsCredentialDelegation credentialDelegation) { @@ -1029,20 +1036,28 @@ public void commitTransaction(CommitTransactionRequest commitTransactionRequest) commitTransactionRequest.tableChanges().stream() .forEach( change -> { - Table table = baseCatalog.loadTable(change.identifier()); + + final Table table = baseCatalog.loadTable(change.identifier()); + if (!(table instanceof BaseTable)) { throw new IllegalStateException( "Cannot wrap catalog that does not produce BaseTable"); } + + TableOperations tableOps = ((BaseTable) table).operations(); + final TableMetadata currentMetadata; + if (baseCatalog instanceof BasePolarisCatalog basePolarisCatalog) { + currentMetadata = basePolarisCatalog.loadTableMetadata(change.identifier()); + } else { + currentMetadata = tableOps.current(); + } + if (isCreate(change)) { throw new BadRequestException( "Unsupported operation: commitTranaction with updateForStagedCreate: %s", change); } - TableOperations tableOps = ((BaseTable) table).operations(); - TableMetadata currentMetadata = tableOps.current(); - // Validate requirements; any CommitFailedExceptions will fail the overall request change.requirements().forEach(requirement -> requirement.validate(currentMetadata)); From c5081d5746568ed2da64323eda9a8bdaead220ea Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 13 Nov 2024 22:24:29 -0800 Subject: [PATCH 31/88] autolint --- .../polaris/service/catalog/PolarisCatalogHandlerWrapper.java | 1 - 1 file changed, 1 deletion(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java index 443c894f98..25623c20c7 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java @@ -1036,7 +1036,6 @@ public void commitTransaction(CommitTransactionRequest commitTransactionRequest) commitTransactionRequest.tableChanges().stream() .forEach( change -> { - final Table table = baseCatalog.loadTable(change.identifier()); if (!(table instanceof BaseTable)) { From 452c5a151d7794044893273021bcbe2cdc975237 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 13 Nov 2024 22:59:20 -0800 Subject: [PATCH 32/88] persist on write --- .../service/catalog/BasePolarisCatalog.java | 39 +++++++++++++------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 8fb2e35f8f..78c2cae7bb 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableMap; import java.io.Closeable; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -1372,23 +1373,37 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { TableLikeEntity entity = TableLikeEntity.of(resolvedEntities == null ? null : resolvedEntities.getRawLeafEntity()); String existingLocation; + long maxMetadataCacheBytes = + callContext + .getPolarisCallContext() + .getConfigurationStore() + .getConfiguration( + callContext.getPolarisCallContext(), PolarisConfiguration.METADATA_CACHE_MAX_BYTES); + String metadataJson = TableMetadataParser.toJson(metadata); + boolean shouldPersistMetadata = + maxMetadataCacheBytes != PolarisConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING && + metadataJson.getBytes(StandardCharsets.UTF_8).length <= maxMetadataCacheBytes; if (null == entity) { existingLocation = null; - entity = - new TableLikeEntity.Builder(tableIdentifier, newLocation) - .setCatalogId(getCatalogId()) - .setSubType(PolarisEntitySubType.TABLE) - .setBaseLocation(metadata.location()) - .setId( - getMetaStoreManager().generateNewEntityId(getCurrentPolarisContext()).getId()) - .build(); + var builder = new TableLikeEntity.Builder(tableIdentifier, newLocation) + .setCatalogId(getCatalogId()) + .setSubType(PolarisEntitySubType.TABLE) + .setBaseLocation(metadata.location()) + .setId( + getMetaStoreManager().generateNewEntityId(getCurrentPolarisContext()).getId()); + if (shouldPersistMetadata) { + builder.setMetadataContent(newLocation, metadataJson); + } + entity = builder.build(); } else { existingLocation = entity.getMetadataLocation(); - entity = - new TableLikeEntity.Builder(entity) + var builder = new TableLikeEntity.Builder(entity) .setBaseLocation(metadata.location()) - .setMetadataLocation(newLocation) - .build(); + .setMetadataLocation(newLocation); + if (shouldPersistMetadata) { + builder.setMetadataContent(newLocation, metadataJson); + } + entity = builder.build(); } if (!Objects.equal(existingLocation, oldLocation)) { if (null == base) { From b85ad92d3526bb167633dc02b11bac8fac842890 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 13 Nov 2024 22:59:24 -0800 Subject: [PATCH 33/88] autolint --- .../service/catalog/BasePolarisCatalog.java | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 78c2cae7bb..e510191392 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -1378,26 +1378,29 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { .getPolarisCallContext() .getConfigurationStore() .getConfiguration( - callContext.getPolarisCallContext(), PolarisConfiguration.METADATA_CACHE_MAX_BYTES); + callContext.getPolarisCallContext(), + PolarisConfiguration.METADATA_CACHE_MAX_BYTES); String metadataJson = TableMetadataParser.toJson(metadata); boolean shouldPersistMetadata = - maxMetadataCacheBytes != PolarisConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING && - metadataJson.getBytes(StandardCharsets.UTF_8).length <= maxMetadataCacheBytes; + maxMetadataCacheBytes != PolarisConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING + && metadataJson.getBytes(StandardCharsets.UTF_8).length <= maxMetadataCacheBytes; if (null == entity) { existingLocation = null; - var builder = new TableLikeEntity.Builder(tableIdentifier, newLocation) - .setCatalogId(getCatalogId()) - .setSubType(PolarisEntitySubType.TABLE) - .setBaseLocation(metadata.location()) - .setId( - getMetaStoreManager().generateNewEntityId(getCurrentPolarisContext()).getId()); + var builder = + new TableLikeEntity.Builder(tableIdentifier, newLocation) + .setCatalogId(getCatalogId()) + .setSubType(PolarisEntitySubType.TABLE) + .setBaseLocation(metadata.location()) + .setId( + getMetaStoreManager().generateNewEntityId(getCurrentPolarisContext()).getId()); if (shouldPersistMetadata) { builder.setMetadataContent(newLocation, metadataJson); } entity = builder.build(); } else { existingLocation = entity.getMetadataLocation(); - var builder = new TableLikeEntity.Builder(entity) + var builder = + new TableLikeEntity.Builder(entity) .setBaseLocation(metadata.location()) .setMetadataLocation(newLocation); if (shouldPersistMetadata) { From fde3910e86c58e05436cb4e4e69c9e0734df6528 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 14 Nov 2024 12:53:47 -0800 Subject: [PATCH 34/88] debugging --- .../service/catalog/BasePolarisCatalog.java | 17 +++++++---------- .../catalog/PolarisCatalogHandlerWrapper.java | 4 +++- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index e510191392..b2258e9c82 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -1384,30 +1384,27 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { boolean shouldPersistMetadata = maxMetadataCacheBytes != PolarisConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING && metadataJson.getBytes(StandardCharsets.UTF_8).length <= maxMetadataCacheBytes; + final TableLikeEntity.Builder builder; if (null == entity) { existingLocation = null; - var builder = + builder = new TableLikeEntity.Builder(tableIdentifier, newLocation) .setCatalogId(getCatalogId()) .setSubType(PolarisEntitySubType.TABLE) .setBaseLocation(metadata.location()) .setId( getMetaStoreManager().generateNewEntityId(getCurrentPolarisContext()).getId()); - if (shouldPersistMetadata) { - builder.setMetadataContent(newLocation, metadataJson); - } - entity = builder.build(); } else { existingLocation = entity.getMetadataLocation(); - var builder = + builder = new TableLikeEntity.Builder(entity) .setBaseLocation(metadata.location()) .setMetadataLocation(newLocation); - if (shouldPersistMetadata) { - builder.setMetadataContent(newLocation, metadataJson); - } - entity = builder.build(); } + if (shouldPersistMetadata) { + builder.setMetadataContent(newLocation, metadataJson); + } + entity = builder.build(); if (!Objects.equal(existingLocation, oldLocation)) { if (null == base) { throw new AlreadyExistsException("Table already exists: %s", tableName()); diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java index 25623c20c7..e829e5cdd2 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java @@ -615,7 +615,9 @@ public LoadTableResponse createTableDirectWithWriteDelegation( final TableMetadata tableMetadata; if (baseCatalog instanceof BasePolarisCatalog basePolarisCatalog) { - tableMetadata = basePolarisCatalog.loadTableMetadata(tableIdentifier); + // tableMetadata = basePolarisCatalog.loadTableMetadata(tableIdentifier); + // TODO debug failures for diffAgainstSingleTable + tableMetadata = baseTable.operations().current(); } else { tableMetadata = baseTable.operations().current(); } From 34b9b0fe245ef3b6548acf3b29816852ae8dde03 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 14 Nov 2024 13:11:49 -0800 Subject: [PATCH 35/88] fix failing test, add TODO --- .../service/catalog/PolarisCatalogHandlerWrapper.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java index e829e5cdd2..0bf2b60cc0 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java @@ -615,9 +615,7 @@ public LoadTableResponse createTableDirectWithWriteDelegation( final TableMetadata tableMetadata; if (baseCatalog instanceof BasePolarisCatalog basePolarisCatalog) { - // tableMetadata = basePolarisCatalog.loadTableMetadata(tableIdentifier); - // TODO debug failures for diffAgainstSingleTable - tableMetadata = baseTable.operations().current(); + tableMetadata = basePolarisCatalog.loadTableMetadata(tableIdentifier); } else { tableMetadata = baseTable.operations().current(); } @@ -1048,7 +1046,9 @@ public void commitTransaction(CommitTransactionRequest commitTransactionRequest) TableOperations tableOps = ((BaseTable) table).operations(); final TableMetadata currentMetadata; if (baseCatalog instanceof BasePolarisCatalog basePolarisCatalog) { - currentMetadata = basePolarisCatalog.loadTableMetadata(change.identifier()); + // TODO use cached metadata once the check in BaseMetastoreTableOperations is fixed + // currentMetadata = basePolarisCatalog.loadTableMetadata(change.identifier()); + currentMetadata = tableOps.current(); } else { currentMetadata = tableOps.current(); } From 6bc07c8c0f3b63af0f90291a638e732c52542aab Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 14 Nov 2024 13:19:58 -0800 Subject: [PATCH 36/88] debugging --- .../service/catalog/BasePolarisCatalog.java | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index b2258e9c82..eaccc1d316 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -1426,6 +1426,35 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { } } + @Override + public void commit(TableMetadata base, TableMetadata metadata) { + // if the metadata is already out of date, reject it + if (base.metadataFileLocation() != current().metadataFileLocation()) { + if (base != null) { + throw new CommitFailedException("Cannot commit: stale table metadata"); + } else { + // when current is non-null, the table exists. but when base is null, the commit is trying + // to create the table + throw new AlreadyExistsException("Table already exists: %s", tableName()); + } + } + // if the metadata is not changed, return early + if (base == metadata) { + LOGGER.info("Nothing to commit."); + return; + } + + long start = System.currentTimeMillis(); + doCommit(base, metadata); + deleteRemovedMetadataFiles(base, metadata); + requestRefresh(); + + LOGGER.info( + "Successfully committed to table {} in {} ms", + tableName(), + System.currentTimeMillis() - start); + } + @Override public FileIO io() { return tableFileIO; From a9dcaa1ac6b4db29c0b9ce9b8970cac29bf2ae59 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 14 Nov 2024 14:06:11 -0800 Subject: [PATCH 37/88] fix current metadata check --- .../service/catalog/BasePolarisCatalog.java | 76 ++++++++++++++++++- 1 file changed, 72 insertions(+), 4 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index eaccc1d316..f609703155 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -36,10 +36,14 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; + +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.iceberg.BaseMetastoreTableOperations; import org.apache.iceberg.BaseTable; @@ -49,6 +53,7 @@ import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableMetadataParser; import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TableProperties; import org.apache.iceberg.aws.s3.S3FileIOProperties; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.SupportsNamespaces; @@ -67,7 +72,9 @@ import org.apache.iceberg.io.CloseableGroup; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.SupportsBulkOperations; import org.apache.iceberg.util.PropertyUtil; +import org.apache.iceberg.util.Tasks; import org.apache.iceberg.view.BaseMetastoreViewCatalog; import org.apache.iceberg.view.BaseViewOperations; import org.apache.iceberg.view.ViewBuilder; @@ -1226,6 +1233,8 @@ private class BasePolarisTableOperations extends BaseMetastoreTableOperations { private final String fullTableName; private FileIO tableFileIO; + private ReentrantLock currentMetadataLock = new ReentrantLock(); + BasePolarisTableOperations(FileIO defaultFileIO, TableIdentifier tableIdentifier) { LOGGER.debug("new BasePolarisTableOperations for {}", tableIdentifier); this.tableIdentifier = tableIdentifier; @@ -1426,18 +1435,31 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { } } + /** + * COPIED FROM {@link BaseMetastoreTableOperations} but without the requirement that base == current() + * + * @param base table metadata on which changes were based + * @param metadata new table metadata with updates + */ @Override public void commit(TableMetadata base, TableMetadata metadata) { + TableMetadata currentMetadata = current(); + // if the metadata is already out of date, reject it - if (base.metadataFileLocation() != current().metadataFileLocation()) { - if (base != null) { - throw new CommitFailedException("Cannot commit: stale table metadata"); - } else { + if (base == null) { + if (currentMetadata != null) { // when current is non-null, the table exists. but when base is null, the commit is trying // to create the table throw new AlreadyExistsException("Table already exists: %s", tableName()); } + } else if (base.metadataFileLocation() != null && + !base.metadataFileLocation().equals(currentMetadata.metadataFileLocation())) { + throw new CommitFailedException("Cannot commit: stale table metadata"); + } else if (base != currentMetadata) { + // This branch is different from BaseMetastoreTableOperations + LOGGER.debug("Base object differs from current metadata; proceeding because locations match"); } + // if the metadata is not changed, return early if (base == metadata) { LOGGER.info("Nothing to commit."); @@ -1455,6 +1477,52 @@ public void commit(TableMetadata base, TableMetadata metadata) { System.currentTimeMillis() - start); } + /** + * + * COPIED FROM {@link BaseMetastoreTableOperations} + * + * Deletes the oldest metadata files if {@link + * TableProperties#METADATA_DELETE_AFTER_COMMIT_ENABLED} is true. + * + * @param base table metadata on which previous versions were based + * @param metadata new table metadata with updated previous versions + */ + protected void deleteRemovedMetadataFiles(TableMetadata base, TableMetadata metadata) { + if (base == null) { + return; + } + + boolean deleteAfterCommit = + metadata.propertyAsBoolean( + TableProperties.METADATA_DELETE_AFTER_COMMIT_ENABLED, + TableProperties.METADATA_DELETE_AFTER_COMMIT_ENABLED_DEFAULT); + + if (deleteAfterCommit) { + Set removedPreviousMetadataFiles = + Sets.newHashSet(base.previousFiles()); + // TableMetadata#addPreviousFile builds up the metadata log and uses + // TableProperties.METADATA_PREVIOUS_VERSIONS_MAX to determine how many files should stay in + // the log, thus we don't include metadata.previousFiles() for deletion - everything else can + // be removed + removedPreviousMetadataFiles.removeAll(metadata.previousFiles()); + if (io() instanceof SupportsBulkOperations) { + ((SupportsBulkOperations) io()) + .deleteFiles( + Iterables.transform( + removedPreviousMetadataFiles, TableMetadata.MetadataLogEntry::file)); + } else { + Tasks.foreach(removedPreviousMetadataFiles) + .noRetry() + .suppressFailureWhenFinished() + .onFailure( + (previousMetadataFile, exc) -> + LOGGER.warn( + "Delete failed for previous metadata file: {}", previousMetadataFile, exc)) + .run(previousMetadataFile -> io().deleteFile(previousMetadataFile.file())); + } + } + } + @Override public FileIO io() { return tableFileIO; From 9b66016c5aef0e9c31b39b4c4249cd3ebf1460f2 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 14 Nov 2024 14:06:14 -0800 Subject: [PATCH 38/88] autolint --- .../service/catalog/BasePolarisCatalog.java | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index f609703155..0276602de0 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -23,6 +23,8 @@ import com.google.common.base.Objects; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; import java.io.Closeable; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -41,9 +43,6 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; - -import com.google.common.collect.Iterables; -import com.google.common.collect.Sets; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.iceberg.BaseMetastoreTableOperations; import org.apache.iceberg.BaseTable; @@ -1436,7 +1435,8 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { } /** - * COPIED FROM {@link BaseMetastoreTableOperations} but without the requirement that base == current() + * COPIED FROM {@link BaseMetastoreTableOperations} but without the requirement that base == + * current() * * @param base table metadata on which changes were based * @param metadata new table metadata with updates @@ -1452,12 +1452,13 @@ public void commit(TableMetadata base, TableMetadata metadata) { // to create the table throw new AlreadyExistsException("Table already exists: %s", tableName()); } - } else if (base.metadataFileLocation() != null && - !base.metadataFileLocation().equals(currentMetadata.metadataFileLocation())) { + } else if (base.metadataFileLocation() != null + && !base.metadataFileLocation().equals(currentMetadata.metadataFileLocation())) { throw new CommitFailedException("Cannot commit: stale table metadata"); } else if (base != currentMetadata) { // This branch is different from BaseMetastoreTableOperations - LOGGER.debug("Base object differs from current metadata; proceeding because locations match"); + LOGGER.debug( + "Base object differs from current metadata; proceeding because locations match"); } // if the metadata is not changed, return early @@ -1478,10 +1479,9 @@ public void commit(TableMetadata base, TableMetadata metadata) { } /** - * * COPIED FROM {@link BaseMetastoreTableOperations} * - * Deletes the oldest metadata files if {@link + *

Deletes the oldest metadata files if {@link * TableProperties#METADATA_DELETE_AFTER_COMMIT_ENABLED} is true. * * @param base table metadata on which previous versions were based @@ -1502,7 +1502,8 @@ protected void deleteRemovedMetadataFiles(TableMetadata base, TableMetadata meta Sets.newHashSet(base.previousFiles()); // TableMetadata#addPreviousFile builds up the metadata log and uses // TableProperties.METADATA_PREVIOUS_VERSIONS_MAX to determine how many files should stay in - // the log, thus we don't include metadata.previousFiles() for deletion - everything else can + // the log, thus we don't include metadata.previousFiles() for deletion - everything else + // can // be removed removedPreviousMetadataFiles.removeAll(metadata.previousFiles()); if (io() instanceof SupportsBulkOperations) { @@ -1517,7 +1518,9 @@ protected void deleteRemovedMetadataFiles(TableMetadata base, TableMetadata meta .onFailure( (previousMetadataFile, exc) -> LOGGER.warn( - "Delete failed for previous metadata file: {}", previousMetadataFile, exc)) + "Delete failed for previous metadata file: {}", + previousMetadataFile, + exc)) .run(previousMetadataFile -> io().deleteFile(previousMetadataFile.file())); } } From 52b82f77dc93fbc6a9749abc00d46416c480fd92 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 14 Nov 2024 14:08:03 -0800 Subject: [PATCH 39/88] fix --- .../polaris/service/catalog/PolarisCatalogHandlerWrapper.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java index 0bf2b60cc0..25623c20c7 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java @@ -1046,9 +1046,7 @@ public void commitTransaction(CommitTransactionRequest commitTransactionRequest) TableOperations tableOps = ((BaseTable) table).operations(); final TableMetadata currentMetadata; if (baseCatalog instanceof BasePolarisCatalog basePolarisCatalog) { - // TODO use cached metadata once the check in BaseMetastoreTableOperations is fixed - // currentMetadata = basePolarisCatalog.loadTableMetadata(change.identifier()); - currentMetadata = tableOps.current(); + currentMetadata = basePolarisCatalog.loadTableMetadata(change.identifier()); } else { currentMetadata = tableOps.current(); } From ee3f401b6a0bb051c53d7a30ed04439bafa5e62a Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 14 Nov 2024 18:08:09 -0800 Subject: [PATCH 40/88] one change per review --- .../apache/polaris/service/catalog/BasePolarisCatalog.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 0276602de0..a72886739e 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -1459,10 +1459,8 @@ public void commit(TableMetadata base, TableMetadata metadata) { // This branch is different from BaseMetastoreTableOperations LOGGER.debug( "Base object differs from current metadata; proceeding because locations match"); - } - - // if the metadata is not changed, return early - if (base == metadata) { + } else if (base.metadataFileLocation().equals(metadata.metadataFileLocation())) { + // if the metadata is not changed, return early LOGGER.info("Nothing to commit."); return; } From 69c2ca516fe77749a001c404b4cbe3427dd626ea Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Sat, 16 Nov 2024 20:16:57 -0800 Subject: [PATCH 41/88] fixes per review --- .../apache/polaris/service/catalog/BasePolarisCatalog.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index a72886739e..0c7861257c 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -38,7 +38,6 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.ReentrantLock; import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -1232,8 +1231,6 @@ private class BasePolarisTableOperations extends BaseMetastoreTableOperations { private final String fullTableName; private FileIO tableFileIO; - private ReentrantLock currentMetadataLock = new ReentrantLock(); - BasePolarisTableOperations(FileIO defaultFileIO, TableIdentifier tableIdentifier) { LOGGER.debug("new BasePolarisTableOperations for {}", tableIdentifier); this.tableIdentifier = tableIdentifier; @@ -1477,7 +1474,9 @@ public void commit(TableMetadata base, TableMetadata metadata) { } /** - * COPIED FROM {@link BaseMetastoreTableOperations} + * COPIED FROM {@link BaseMetastoreTableOperations} as the method is private there + * This is moved to `CatalogUtils` in Iceberg 1.7.0 and can be called from there once + * we depend on Iceberg 1.7.0 * *

Deletes the oldest metadata files if {@link * TableProperties#METADATA_DELETE_AFTER_COMMIT_ENABLED} is true. From 7b3e68d4eed88c8f117965a0537899b3fdc97969 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Sat, 16 Nov 2024 20:17:03 -0800 Subject: [PATCH 42/88] autolint --- .../apache/polaris/service/catalog/BasePolarisCatalog.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 0c7861257c..8d75f9c64a 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -1474,9 +1474,9 @@ public void commit(TableMetadata base, TableMetadata metadata) { } /** - * COPIED FROM {@link BaseMetastoreTableOperations} as the method is private there - * This is moved to `CatalogUtils` in Iceberg 1.7.0 and can be called from there once - * we depend on Iceberg 1.7.0 + * COPIED FROM {@link BaseMetastoreTableOperations} as the method is private there This is moved + * to `CatalogUtils` in Iceberg 1.7.0 and can be called from there once we depend on Iceberg + * 1.7.0 * *

Deletes the oldest metadata files if {@link * TableProperties#METADATA_DELETE_AFTER_COMMIT_ENABLED} is true. From 0b5291e14e3be6750d0319cae73268c346c6fd93 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Tue, 19 Nov 2024 11:18:05 -0800 Subject: [PATCH 43/88] add a comment per review; make check more clear --- .../java/org/apache/polaris/core/entity/TableLikeEntity.java | 3 ++- .../polaris/service/persistence/MetadataCacheManager.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/TableLikeEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/TableLikeEntity.java index 1d6b67dc12..c3aa403e4a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/TableLikeEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/TableLikeEntity.java @@ -34,7 +34,8 @@ public class TableLikeEntity extends PolarisEntity { private static final String METADATA_CACHE_CONTENT_KEY = "metadata-cache-content"; // For applicable types, this key on the "internalProperties" map will return the location of the - // `metadata.json` that is cached in METADATA_CACHE_CONTENT_KEY + // `metadata.json` that is cached in `METADATA_CACHE_CONTENT_KEY`. This will often match the + // current metadata location in `METADATA_LOCATION_KEY`; if it does not the cache is invalid private static final String METADATA_CACHE_LOCATION_KEY = "metadata-cache-location"; public static final String USER_SPECIFIED_WRITE_DATA_LOCATION_KEY = "write.data.path"; diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index f9712a70b3..2c64197cb7 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -53,7 +53,7 @@ public static TableMetadata loadTableMetadata( PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getResolvedPath(tableIdentifier, PolarisEntitySubType.TABLE); TableLikeEntity tableLikeEntity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); - boolean isCacheValid = + boolean isCacheValid = tableLikeEntity.getMetadataLocation() != null && tableLikeEntity.getMetadataLocation().equals(tableLikeEntity.getMetadataCacheLocationKey()); if (isCacheValid) { LOGGER.debug(String.format("Using cached metadata for %s", tableIdentifier)); From 5488d76652b3f4b5e968cdf0017fe52fc1d24944 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Tue, 19 Nov 2024 11:18:10 -0800 Subject: [PATCH 44/88] autolint --- .../polaris/service/persistence/MetadataCacheManager.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 2c64197cb7..a0a7b934e4 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -53,8 +53,11 @@ public static TableMetadata loadTableMetadata( PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getResolvedPath(tableIdentifier, PolarisEntitySubType.TABLE); TableLikeEntity tableLikeEntity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); - boolean isCacheValid = tableLikeEntity.getMetadataLocation() != null && - tableLikeEntity.getMetadataLocation().equals(tableLikeEntity.getMetadataCacheLocationKey()); + boolean isCacheValid = + tableLikeEntity.getMetadataLocation() != null + && tableLikeEntity + .getMetadataLocation() + .equals(tableLikeEntity.getMetadataCacheLocationKey()); if (isCacheValid) { LOGGER.debug(String.format("Using cached metadata for %s", tableIdentifier)); return TableMetadataParser.fromJson(tableLikeEntity.getMetadataCacheContent()); From 64e8d2bcc37b11e4ec1b961b5c3b9ae4be2a454f Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Tue, 19 Nov 2024 11:19:47 -0800 Subject: [PATCH 45/88] fix method name --- .../java/org/apache/polaris/core/entity/TableLikeEntity.java | 2 +- .../polaris/service/persistence/MetadataCacheManager.java | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/TableLikeEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/TableLikeEntity.java index c3aa403e4a..c38b2d95ee 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/TableLikeEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/TableLikeEntity.java @@ -82,7 +82,7 @@ public String getMetadataCacheContent() { } @JsonIgnore - public String getMetadataCacheLocationKey() { + public String getMetadataCacheLocation() { return getInternalPropertiesAsMap().get(METADATA_CACHE_LOCATION_KEY); } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index a0a7b934e4..1c36720435 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -54,10 +54,9 @@ public static TableMetadata loadTableMetadata( resolvedEntityView.getResolvedPath(tableIdentifier, PolarisEntitySubType.TABLE); TableLikeEntity tableLikeEntity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); boolean isCacheValid = - tableLikeEntity.getMetadataLocation() != null - && tableLikeEntity + tableLikeEntity.getMetadataCacheLocation() != null&& tableLikeEntity .getMetadataLocation() - .equals(tableLikeEntity.getMetadataCacheLocationKey()); + .equals(tableLikeEntity.getMetadataCacheLocation()); if (isCacheValid) { LOGGER.debug(String.format("Using cached metadata for %s", tableIdentifier)); return TableMetadataParser.fromJson(tableLikeEntity.getMetadataCacheContent()); From 5eca203f84588853b10038ca7ee305b986b77337 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Tue, 19 Nov 2024 11:19:51 -0800 Subject: [PATCH 46/88] autolint --- .../polaris/service/persistence/MetadataCacheManager.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 1c36720435..e0ffc68dc7 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -54,7 +54,8 @@ public static TableMetadata loadTableMetadata( resolvedEntityView.getResolvedPath(tableIdentifier, PolarisEntitySubType.TABLE); TableLikeEntity tableLikeEntity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); boolean isCacheValid = - tableLikeEntity.getMetadataCacheLocation() != null&& tableLikeEntity + tableLikeEntity.getMetadataCacheLocation() != null + && tableLikeEntity .getMetadataLocation() .equals(tableLikeEntity.getMetadataCacheLocation()); if (isCacheValid) { From 7f89a244801fb79fb1c715e7a476eb3f67ff242c Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 21 Nov 2024 11:21:35 -0800 Subject: [PATCH 47/88] remove check --- .../polaris/core/PolarisConfiguration.java | 13 +++++++---- .../service/catalog/BasePolarisCatalog.java | 11 +++++---- .../persistence/MetadataCacheManager.java | 8 ++----- .../catalog/BasePolarisCatalogTest.java | 23 ++++--------------- 4 files changed, 22 insertions(+), 33 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java index 8a67132212..30d395ae6f 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java @@ -227,13 +227,16 @@ public static Builder builder() { .defaultValue(true) .build(); - public static final Long METADATA_CACHE_MAX_BYTES_NO_CACHING = 0L; - public static final PolarisConfiguration METADATA_CACHE_MAX_BYTES = - PolarisConfiguration.builder() + public static final int METADATA_CACHE_MAX_BYTES_NO_CACHING = 0; + public static final int METADATA_CACHE_MAX_BYTES_INFINITE_CACHING = -1; + public static final PolarisConfiguration METADATA_CACHE_MAX_BYTES = + PolarisConfiguration.builder() .key("METADATA_CACHE_MAX_BYTES") + .catalogConfig("metadata.cache.max.bytes") .description( - "If nonzero, the max size a table's metadata can be in order to be cached in the persistence layer." - + " If zero, no metadata will be cached or served from the cache. If -1, all metadata will be cached.") + "If nonzero, the approximate max size a table's metadata can be in order to be cached in the persistence" + + " layer. If zero, no metadata will be cached or served from the cache. If -1, all metadata" + + " will be cached.") .defaultValue(METADATA_CACHE_MAX_BYTES_NO_CACHING) .validation(value -> value >= -1) .build(); diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 8d75f9c64a..d9e80334d0 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -1378,17 +1378,20 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { TableLikeEntity entity = TableLikeEntity.of(resolvedEntities == null ? null : resolvedEntities.getRawLeafEntity()); String existingLocation; - long maxMetadataCacheBytes = + int maxMetadataCacheBytes = callContext .getPolarisCallContext() .getConfigurationStore() .getConfiguration( callContext.getPolarisCallContext(), + catalogEntity, PolarisConfiguration.METADATA_CACHE_MAX_BYTES); String metadataJson = TableMetadataParser.toJson(metadata); - boolean shouldPersistMetadata = - maxMetadataCacheBytes != PolarisConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING - && metadataJson.getBytes(StandardCharsets.UTF_8).length <= maxMetadataCacheBytes; + boolean shouldPersistMetadata = switch (maxMetadataCacheBytes) { + case PolarisConfiguration.METADATA_CACHE_MAX_BYTES_INFINITE_CACHING -> true; + case PolarisConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING -> false; + default -> metadataJson.length() <= maxMetadataCacheBytes; + }; final TableLikeEntity.Builder builder; if (null == entity) { existingLocation = null; diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index e0ffc68dc7..6af5a62837 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -53,12 +53,8 @@ public static TableMetadata loadTableMetadata( PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getResolvedPath(tableIdentifier, PolarisEntitySubType.TABLE); TableLikeEntity tableLikeEntity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); - boolean isCacheValid = - tableLikeEntity.getMetadataCacheLocation() != null - && tableLikeEntity - .getMetadataLocation() - .equals(tableLikeEntity.getMetadataCacheLocation()); - if (isCacheValid) { + String cacheContent = tableLikeEntity.getMetadataCacheContent(); + if (cacheContent != null) { LOGGER.debug(String.format("Using cached metadata for %s", tableIdentifier)); return TableMetadataParser.fromJson(tableLikeEntity.getMetadataCacheContent()); } else { diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java index 5f9b272195..ac09d3c649 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java @@ -206,6 +206,8 @@ public void before() { PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") .addProperty( PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") + .addProperty( + PolarisConfiguration.METADATA_CACHE_MAX_BYTES.catalogConfig(), "-1") .setStorageConfigurationInfo(storageConfigModel, storageLocation) .build()); @@ -1547,18 +1549,6 @@ public void testMetadataCachingWithManualFallback() { Table createdTable = catalog.createTable(tableIdentifier, schema); TableMetadata originalMetadata = ((BaseTable) createdTable).operations().current(); - TableMetadata loadedMetadata = - MetadataCacheManager.loadTableMetadata( - tableIdentifier, - Long.MAX_VALUE, - polarisContext, - metaStoreManager, - passthroughView, - () -> originalMetadata); - - // The first time, the fallback is called - Assertions.assertThat(loadedMetadata).isSameAs(originalMetadata); - TableMetadata cachedMetadata = MetadataCacheManager.loadTableMetadata( tableIdentifier, @@ -1570,18 +1560,17 @@ public void testMetadataCachingWithManualFallback() { throw new IllegalStateException("Fell back even though a cache entry should exist!"); }); - // The second time, it's loaded from the cache + // The metadata object is loaded from the cache Assertions.assertThat(cachedMetadata).isNotSameAs(originalMetadata); // The content should match what was cached Assertions.assertThat(TableMetadataParser.toJson(cachedMetadata)) - .isEqualTo(TableMetadataParser.toJson(loadedMetadata)); + .isEqualTo(TableMetadataParser.toJson(originalMetadata)); // Update the table TableOperations tableOps = catalog.newTableOps(tableIdentifier); TableMetadata updatedMetadata = tableOps.current().updateSchema(buildSchema(100), 100); tableOps.commit(tableOps.current(), updatedMetadata); - AtomicBoolean wasFallbackCalledAgain = new AtomicBoolean(false); // Read from the cache; it should detect a chance due to the update and load the new fallback TableMetadata reloadedMetadata = @@ -1592,12 +1581,10 @@ public void testMetadataCachingWithManualFallback() { metaStoreManager, passthroughView, () -> { - wasFallbackCalledAgain.set(true); - return updatedMetadata; + throw new IllegalStateException("Fell back even though a cache entry should be updated on write"); }); Assertions.assertThat(reloadedMetadata).isNotSameAs(cachedMetadata); Assertions.assertThat(reloadedMetadata.schema().columns().size()).isEqualTo(100); - Assertions.assertThat(wasFallbackCalledAgain.get()).isTrue(); } } From 9829486c6bd179e6eafd429a6f728078543c7563 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 21 Nov 2024 11:21:46 -0800 Subject: [PATCH 48/88] autolint --- .../apache/polaris/core/PolarisConfiguration.java | 6 +++--- .../polaris/service/catalog/BasePolarisCatalog.java | 12 ++++++------ .../service/catalog/BasePolarisCatalogTest.java | 7 +++---- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java index 30d395ae6f..30d5da71ab 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java @@ -234,9 +234,9 @@ public static Builder builder() { .key("METADATA_CACHE_MAX_BYTES") .catalogConfig("metadata.cache.max.bytes") .description( - "If nonzero, the approximate max size a table's metadata can be in order to be cached in the persistence" + - " layer. If zero, no metadata will be cached or served from the cache. If -1, all metadata" + - " will be cached.") + "If nonzero, the approximate max size a table's metadata can be in order to be cached in the persistence" + + " layer. If zero, no metadata will be cached or served from the cache. If -1, all metadata" + + " will be cached.") .defaultValue(METADATA_CACHE_MAX_BYTES_NO_CACHING) .validation(value -> value >= -1) .build(); diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index d9e80334d0..2cb34c87ce 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -27,7 +27,6 @@ import com.google.common.collect.Sets; import java.io.Closeable; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -1387,11 +1386,12 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { catalogEntity, PolarisConfiguration.METADATA_CACHE_MAX_BYTES); String metadataJson = TableMetadataParser.toJson(metadata); - boolean shouldPersistMetadata = switch (maxMetadataCacheBytes) { - case PolarisConfiguration.METADATA_CACHE_MAX_BYTES_INFINITE_CACHING -> true; - case PolarisConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING -> false; - default -> metadataJson.length() <= maxMetadataCacheBytes; - }; + boolean shouldPersistMetadata = + switch (maxMetadataCacheBytes) { + case PolarisConfiguration.METADATA_CACHE_MAX_BYTES_INFINITE_CACHING -> true; + case PolarisConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING -> false; + default -> metadataJson.length() <= maxMetadataCacheBytes; + }; final TableLikeEntity.Builder builder; if (null == entity) { existingLocation = null; diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java index ac09d3c649..b7339d0d1f 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java @@ -34,7 +34,6 @@ import java.util.Map; import java.util.Set; import java.util.UUID; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; import org.apache.commons.lang3.NotImplementedException; import org.apache.iceberg.BaseTable; @@ -206,8 +205,7 @@ public void before() { PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") .addProperty( PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .addProperty( - PolarisConfiguration.METADATA_CACHE_MAX_BYTES.catalogConfig(), "-1") + .addProperty(PolarisConfiguration.METADATA_CACHE_MAX_BYTES.catalogConfig(), "-1") .setStorageConfigurationInfo(storageConfigModel, storageLocation) .build()); @@ -1581,7 +1579,8 @@ public void testMetadataCachingWithManualFallback() { metaStoreManager, passthroughView, () -> { - throw new IllegalStateException("Fell back even though a cache entry should be updated on write"); + throw new IllegalStateException( + "Fell back even though a cache entry should be updated on write"); }); Assertions.assertThat(reloadedMetadata).isNotSameAs(cachedMetadata); From 73f5dd2c59082dcaad986b4ba8c812b0296db6ea Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 25 Nov 2024 11:52:37 -0800 Subject: [PATCH 49/88] defer serialization --- .../apache/polaris/service/catalog/BasePolarisCatalog.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 2cb34c87ce..778ad102a5 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -1385,12 +1385,15 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { callContext.getPolarisCallContext(), catalogEntity, PolarisConfiguration.METADATA_CACHE_MAX_BYTES); - String metadataJson = TableMetadataParser.toJson(metadata); + String metadataJson = null; boolean shouldPersistMetadata = switch (maxMetadataCacheBytes) { case PolarisConfiguration.METADATA_CACHE_MAX_BYTES_INFINITE_CACHING -> true; case PolarisConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING -> false; - default -> metadataJson.length() <= maxMetadataCacheBytes; + default -> { + metadataJson = TableMetadataParser.toJson(metadata); + yield metadataJson.length() <= maxMetadataCacheBytes; + } }; final TableLikeEntity.Builder builder; if (null == entity) { From 59361ed0bec6c3a2f8c609b32bbf4a9ffda12e66 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 25 Nov 2024 13:45:01 -0800 Subject: [PATCH 50/88] bounded serde --- .../service/catalog/BasePolarisCatalog.java | 16 +-- .../persistence/MetadataCacheManager.java | 107 ++++++++++++++++-- .../catalog/BasePolarisCatalogTest.java | 4 +- 3 files changed, 109 insertions(+), 18 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 778ad102a5..7505c1c930 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -828,7 +828,7 @@ public Map getCredentialConfig( } public TableMetadata loadTableMetadata(TableIdentifier identifier) { - long maxMetadataCacheBytes = + int maxMetadataCacheBytes = callContext .getPolarisCallContext() .getConfigurationStore() @@ -1385,14 +1385,14 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { callContext.getPolarisCallContext(), catalogEntity, PolarisConfiguration.METADATA_CACHE_MAX_BYTES); - String metadataJson = null; + Optional metadataJsonOpt = Optional.empty(); boolean shouldPersistMetadata = switch (maxMetadataCacheBytes) { case PolarisConfiguration.METADATA_CACHE_MAX_BYTES_INFINITE_CACHING -> true; case PolarisConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING -> false; default -> { - metadataJson = TableMetadataParser.toJson(metadata); - yield metadataJson.length() <= maxMetadataCacheBytes; + metadataJsonOpt = MetadataCacheManager.toBoundedJson(metadata, maxMetadataCacheBytes); + yield metadataJsonOpt.map(String::length).map(l -> l <= maxMetadataCacheBytes).orElse(false); } }; final TableLikeEntity.Builder builder; @@ -1412,8 +1412,8 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { .setBaseLocation(metadata.location()) .setMetadataLocation(newLocation); } - if (shouldPersistMetadata) { - builder.setMetadataContent(newLocation, metadataJson); + if (shouldPersistMetadata && metadataJsonOpt.isPresent()) { + builder.setMetadataContent(newLocation, metadataJsonOpt.get()); } entity = builder.build(); if (!Objects.equal(existingLocation, oldLocation)) { @@ -1438,7 +1438,7 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { } /** - * COPIED FROM {@link BaseMetastoreTableOperations} but without the requirement that base == + * Copied from {@link BaseMetastoreTableOperations} but without the requirement that base == * current() * * @param base table metadata on which changes were based @@ -1480,7 +1480,7 @@ public void commit(TableMetadata base, TableMetadata metadata) { } /** - * COPIED FROM {@link BaseMetastoreTableOperations} as the method is private there This is moved + * Copied from {@link BaseMetastoreTableOperations} as the method is private there This is moved * to `CatalogUtils` in Iceberg 1.7.0 and can be called from there once we depend on Iceberg * 1.7.0 * diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 6af5a62837..105d7101c8 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -18,12 +18,20 @@ */ package org.apache.polaris.service.persistence; +import java.io.IOException; +import java.io.StringWriter; +import java.io.Writer; import java.nio.charset.StandardCharsets; +import java.util.Optional; import java.util.function.Supplier; + +import com.fasterxml.jackson.core.JsonGenerator; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableMetadataParser; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.RuntimeIOException; +import org.apache.iceberg.util.JsonUtil; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisConfiguration; import org.apache.polaris.core.entity.PolarisEntity; @@ -44,7 +52,7 @@ public class MetadataCacheManager { */ public static TableMetadata loadTableMetadata( TableIdentifier tableIdentifier, - long maxBytesToCache, + int maxBytesToCache, PolarisCallContext callContext, PolarisMetaStoreManager metastoreManager, PolarisResolutionManifestCatalogView resolvedEntityView, @@ -74,6 +82,20 @@ public static TableMetadata loadTableMetadata( } } + public static Optional toBoundedJson(TableMetadata metadata, int maxBytes) { + try (StringWriter writer = new StringWriter()) { + BoundedWriter boundedWriter = new BoundedWriter(writer, maxBytes); + JsonGenerator generator = JsonUtil.factory().createGenerator(boundedWriter); + TableMetadataParser.toJson(metadata, generator); + generator.flush(); + return Optional.ofNullable(writer.toString()); + } catch (BoundedWriterException b) { + return Optional.empty(); + } catch (IOException e) { + throw new RuntimeIOException(e, "Failed to write json for: %s", metadata); + } + } + /** * Attempt to add table metadata to the cache * @@ -82,19 +104,18 @@ public static TableMetadata loadTableMetadata( private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( TableLikeEntity tableLikeEntity, TableMetadata metadata, - long maxBytesToCache, + int maxBytesToCache, PolarisCallContext callContext, PolarisMetaStoreManager metaStoreManager, PolarisResolutionManifestCatalogView resolvedEntityView) { - String json = TableMetadataParser.toJson(metadata); + Optional jsonOpt = toBoundedJson(metadata, maxBytesToCache); // We should not reach this method in this case, but check just in case... if (maxBytesToCache != PolarisConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING) { - long sizeInBytes = json.getBytes(StandardCharsets.UTF_8).length; - if (sizeInBytes > maxBytesToCache) { + if (jsonOpt.isEmpty()) { LOGGER.debug( String.format( - "Will not cache metadata for %s; metadata is %d bytes and the limit is %d", - tableLikeEntity.getTableIdentifier(), sizeInBytes, maxBytesToCache)); + "Will not cache metadata for %s; metadata above the limit of %d bytes", + tableLikeEntity.getTableIdentifier(), maxBytesToCache)); return new PolarisMetaStoreManager.EntityResult( PolarisMetaStoreManager.ReturnStatus.SUCCESS, null); } else { @@ -102,7 +123,7 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( String.format("Caching metadata for %s", tableLikeEntity.getTableIdentifier())); TableLikeEntity newTableLikeEntity = new TableLikeEntity.Builder(tableLikeEntity) - .setMetadataContent(tableLikeEntity.getMetadataLocation(), json) + .setMetadataContent(tableLikeEntity.getMetadataLocation(), jsonOpt.get()) .build(); PolarisResolvedPathWrapper resolvedPath = resolvedEntityView.getResolvedPath( @@ -136,4 +157,74 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( PolarisMetaStoreManager.ReturnStatus.SUCCESS, null); } } + + private static class BoundedWriterException extends RuntimeException {} + + private static class BoundedWriter extends Writer { + private final Writer delegate; + private final int maxBytes; + private long writtenBytes = 0; + + public BoundedWriter(Writer delegate, int maxBytes) { + this.delegate = delegate; + this.maxBytes = maxBytes; + } + + private void checkLimit(long bytesToWrite) throws IOException { + if (writtenBytes + bytesToWrite > maxBytes) { + throw new BoundedWriterException(); + } + } + + @Override + public void write(char[] cbuf, int off, int len) throws IOException { + checkLimit(len); + delegate.write(cbuf, off, len); + writtenBytes += len; + } + + @Override + public void write(int c) throws IOException { + checkLimit(1); + delegate.write(c); + writtenBytes++; + } + + @Override + public void write(String str, int off, int len) throws IOException { + checkLimit(len); + delegate.write(str, off, len); + writtenBytes += len; + } + + @Override + public Writer append(CharSequence csq) throws IOException { + String str = (csq == null) ? "null" : csq.toString(); + write(str, 0, str.length()); + return this; + } + + @Override + public Writer append(CharSequence csq, int start, int end) throws IOException { + String str = (csq == null) ? "null" : csq.subSequence(start, end).toString(); + write(str, 0, str.length()); + return this; + } + + @Override + public Writer append(char c) throws IOException { + write(c); + return this; + } + + @Override + public void flush() throws IOException { + delegate.flush(); + } + + @Override + public void close() throws IOException { + delegate.close(); + } + } } diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java index b7339d0d1f..786141320b 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java @@ -1550,7 +1550,7 @@ public void testMetadataCachingWithManualFallback() { TableMetadata cachedMetadata = MetadataCacheManager.loadTableMetadata( tableIdentifier, - Long.MAX_VALUE, + Integer.MAX_VALUE, polarisContext, metaStoreManager, passthroughView, @@ -1574,7 +1574,7 @@ public void testMetadataCachingWithManualFallback() { TableMetadata reloadedMetadata = MetadataCacheManager.loadTableMetadata( tableIdentifier, - Long.MAX_VALUE, + Integer.MAX_VALUE, polarisContext, metaStoreManager, passthroughView, From c67492f3000d24ebf674015094b1bb1a0cf589cb Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 25 Nov 2024 13:45:06 -0800 Subject: [PATCH 51/88] autolint --- .../apache/polaris/service/catalog/BasePolarisCatalog.java | 5 ++++- .../polaris/service/persistence/MetadataCacheManager.java | 4 +--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 7505c1c930..643bc580fc 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -1392,7 +1392,10 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { case PolarisConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING -> false; default -> { metadataJsonOpt = MetadataCacheManager.toBoundedJson(metadata, maxMetadataCacheBytes); - yield metadataJsonOpt.map(String::length).map(l -> l <= maxMetadataCacheBytes).orElse(false); + yield metadataJsonOpt + .map(String::length) + .map(l -> l <= maxMetadataCacheBytes) + .orElse(false); } }; final TableLikeEntity.Builder builder; diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 105d7101c8..227e7f7455 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -18,14 +18,12 @@ */ package org.apache.polaris.service.persistence; +import com.fasterxml.jackson.core.JsonGenerator; import java.io.IOException; import java.io.StringWriter; import java.io.Writer; -import java.nio.charset.StandardCharsets; import java.util.Optional; import java.util.function.Supplier; - -import com.fasterxml.jackson.core.JsonGenerator; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableMetadataParser; From 85ec25b3da7334ce517d3d46e87f7d337f1aabb5 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 25 Nov 2024 13:54:29 -0800 Subject: [PATCH 52/88] no more exceptions --- .../persistence/MetadataCacheManager.java | 64 +++++++++++-------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 227e7f7455..631e8858ee 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -80,15 +80,19 @@ public static TableMetadata loadTableMetadata( } } + /** */ public static Optional toBoundedJson(TableMetadata metadata, int maxBytes) { - try (StringWriter writer = new StringWriter()) { - BoundedWriter boundedWriter = new BoundedWriter(writer, maxBytes); + try (StringWriter unboundedWriter = new StringWriter()) { + BoundedWriter boundedWriter = new BoundedWriter(unboundedWriter, maxBytes); JsonGenerator generator = JsonUtil.factory().createGenerator(boundedWriter); TableMetadataParser.toJson(metadata, generator); generator.flush(); - return Optional.ofNullable(writer.toString()); - } catch (BoundedWriterException b) { - return Optional.empty(); + String result = boundedWriter.toString(); + if (boundedWriter.isLimitExceeded()) { + return Optional.empty(); + } else { + return Optional.ofNullable(result); + } } catch (IOException e) { throw new RuntimeIOException(e, "Failed to write json for: %s", metadata); } @@ -156,72 +160,80 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( } } - private static class BoundedWriterException extends RuntimeException {} - private static class BoundedWriter extends Writer { private final Writer delegate; private final int maxBytes; private long writtenBytes = 0; + private boolean limitExceeded = false; public BoundedWriter(Writer delegate, int maxBytes) { this.delegate = delegate; this.maxBytes = maxBytes; } - private void checkLimit(long bytesToWrite) throws IOException { + private boolean canWriteBytes(long bytesToWrite) throws IOException { if (writtenBytes + bytesToWrite > maxBytes) { - throw new BoundedWriterException(); + limitExceeded = true; } + return limitExceeded; + } + + /** `true` when the writer was asked to write more than `maxBytes` bytes */ + public final boolean isLimitExceeded() { + return limitExceeded; } @Override - public void write(char[] cbuf, int off, int len) throws IOException { - checkLimit(len); - delegate.write(cbuf, off, len); - writtenBytes += len; + public final void write(char[] cbuf, int off, int len) throws IOException { + if (canWriteBytes(len)) { + delegate.write(cbuf, off, len); + writtenBytes += len; + } } @Override - public void write(int c) throws IOException { - checkLimit(1); - delegate.write(c); - writtenBytes++; + public final void write(int c) throws IOException { + if (canWriteBytes(1)) { + delegate.write(c); + writtenBytes++; + } } @Override - public void write(String str, int off, int len) throws IOException { - checkLimit(len); - delegate.write(str, off, len); - writtenBytes += len; + public final void write(String str, int off, int len) throws IOException { + if (canWriteBytes(len)) { + delegate.write(str, off, len); + writtenBytes += len; + } } @Override - public Writer append(CharSequence csq) throws IOException { + public final Writer append(CharSequence csq) throws IOException { String str = (csq == null) ? "null" : csq.toString(); write(str, 0, str.length()); return this; } @Override - public Writer append(CharSequence csq, int start, int end) throws IOException { + public final Writer append(CharSequence csq, int start, int end) throws IOException { String str = (csq == null) ? "null" : csq.subSequence(start, end).toString(); write(str, 0, str.length()); return this; } @Override - public Writer append(char c) throws IOException { + public final Writer append(char c) throws IOException { write(c); return this; } @Override - public void flush() throws IOException { + public final void flush() throws IOException { delegate.flush(); } @Override - public void close() throws IOException { + public final void close() throws IOException { delegate.close(); } } From 424b7d9da3e631c29c17c5836920b956278a5f06 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 25 Nov 2024 13:55:33 -0800 Subject: [PATCH 53/88] comment --- .../polaris/service/persistence/MetadataCacheManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 631e8858ee..186e576708 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -80,7 +80,7 @@ public static TableMetadata loadTableMetadata( } } - /** */ + /** Convert a {@link TableMetadata} to JSON, with the size bounded */ public static Optional toBoundedJson(TableMetadata metadata, int maxBytes) { try (StringWriter unboundedWriter = new StringWriter()) { BoundedWriter boundedWriter = new BoundedWriter(unboundedWriter, maxBytes); From a5fe3c123bb40669095b5b918b9a87eb90188cab Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 25 Nov 2024 13:59:01 -0800 Subject: [PATCH 54/88] fix --- .../polaris/service/persistence/MetadataCacheManager.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 186e576708..37e8b7b376 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -119,7 +119,7 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( "Will not cache metadata for %s; metadata above the limit of %d bytes", tableLikeEntity.getTableIdentifier(), maxBytesToCache)); return new PolarisMetaStoreManager.EntityResult( - PolarisMetaStoreManager.ReturnStatus.SUCCESS, null); + PolarisMetaStoreManager.EntityResult.ReturnStatus.SUCCESS, null); } else { LOGGER.debug( String.format("Caching metadata for %s", tableLikeEntity.getTableIdentifier())); @@ -144,7 +144,7 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( "Encountered an error while caching %s: %s", tableLikeEntity.getTableIdentifier(), e)); return new PolarisMetaStoreManager.EntityResult( - PolarisMetaStoreManager.ReturnStatus.UNEXPECTED_ERROR_SIGNALED, e.getMessage()); + PolarisMetaStoreManager.EntityResult.ReturnStatus.UNEXPECTED_ERROR_SIGNALED, e.getMessage()); } else { throw e; } @@ -156,7 +156,7 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( "Will not cache metadata for %s; metadata caching is disabled", tableLikeEntity.getTableIdentifier())); return new PolarisMetaStoreManager.EntityResult( - PolarisMetaStoreManager.ReturnStatus.SUCCESS, null); + PolarisMetaStoreManager.EntityResult.ReturnStatus.SUCCESS, null); } } From 0b152f783ba426c87858e9c25782aabaec182be1 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 25 Nov 2024 13:59:06 -0800 Subject: [PATCH 55/88] autolint --- .../polaris/service/persistence/MetadataCacheManager.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 37e8b7b376..0927b91376 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -144,7 +144,8 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( "Encountered an error while caching %s: %s", tableLikeEntity.getTableIdentifier(), e)); return new PolarisMetaStoreManager.EntityResult( - PolarisMetaStoreManager.EntityResult.ReturnStatus.UNEXPECTED_ERROR_SIGNALED, e.getMessage()); + PolarisMetaStoreManager.EntityResult.ReturnStatus.UNEXPECTED_ERROR_SIGNALED, + e.getMessage()); } else { throw e; } From 35848427d280bb26ae1bd8d428712e25294ccd7a Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 25 Nov 2024 14:14:09 -0800 Subject: [PATCH 56/88] stable --- .../service/catalog/BasePolarisCatalog.java | 5 ++++- .../persistence/MetadataCacheManager.java | 20 ++++++++++++++----- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 815aadc1b3..8331dbd5ae 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -1390,8 +1390,11 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { Optional metadataJsonOpt = Optional.empty(); boolean shouldPersistMetadata = switch (maxMetadataCacheBytes) { - case PolarisConfiguration.METADATA_CACHE_MAX_BYTES_INFINITE_CACHING -> true; case PolarisConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING -> false; + case PolarisConfiguration.METADATA_CACHE_MAX_BYTES_INFINITE_CACHING -> { + metadataJsonOpt = MetadataCacheManager.toBoundedJson(metadata, maxMetadataCacheBytes); + yield true; + } default -> { metadataJsonOpt = MetadataCacheManager.toBoundedJson(metadata, maxMetadataCacheBytes); yield metadataJsonOpt diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 0927b91376..788f92f4e8 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -83,7 +83,7 @@ public static TableMetadata loadTableMetadata( /** Convert a {@link TableMetadata} to JSON, with the size bounded */ public static Optional toBoundedJson(TableMetadata metadata, int maxBytes) { try (StringWriter unboundedWriter = new StringWriter()) { - BoundedWriter boundedWriter = new BoundedWriter(unboundedWriter, maxBytes); + BoundedStringWriter boundedWriter = new BoundedStringWriter(unboundedWriter, maxBytes); JsonGenerator generator = JsonUtil.factory().createGenerator(boundedWriter); TableMetadataParser.toJson(metadata, generator); generator.flush(); @@ -161,22 +161,27 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( } } - private static class BoundedWriter extends Writer { + private static class BoundedStringWriter extends Writer { private final Writer delegate; private final int maxBytes; private long writtenBytes = 0; private boolean limitExceeded = false; - public BoundedWriter(Writer delegate, int maxBytes) { + /** Create a new BoundedWriter with a given limit `maxBytes`. -1 means no limit. */ + public BoundedStringWriter(StringWriter delegate, int maxBytes) { this.delegate = delegate; - this.maxBytes = maxBytes; + if (maxBytes == -1) { + this.maxBytes = Integer.MAX_VALUE; + } else { + this.maxBytes = maxBytes; + } } private boolean canWriteBytes(long bytesToWrite) throws IOException { if (writtenBytes + bytesToWrite > maxBytes) { limitExceeded = true; } - return limitExceeded; + return !limitExceeded; } /** `true` when the writer was asked to write more than `maxBytes` bytes */ @@ -237,5 +242,10 @@ public final void flush() throws IOException { public final void close() throws IOException { delegate.close(); } + + @Override + public final String toString() { + return delegate.toString(); + } } } From 8d704eb51e3b33b2742b81b88b29470b0fbaf26e Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 25 Nov 2024 14:19:08 -0800 Subject: [PATCH 57/88] fix --- .../java/org/apache/polaris/core/PolarisConfiguration.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java index 460ad36549..1f3c531afa 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java @@ -47,7 +47,8 @@ public PolarisConfiguration( this.typ = (Class) defaultValue.getClass(); this.validation = validation; - validate(cast(defaultValue)); + // Force validation: + cast(defaultValue); } public boolean hasCatalogConfig() { From 37f4d7a72613b39ff3c696a3a035432a22174965 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Tue, 29 Apr 2025 10:22:44 -0700 Subject: [PATCH 58/88] start fixing conflicts --- .../quarkus/catalog/IcebergCatalogTest.java | 78 ++-------- .../catalog/iceberg/IcebergCatalog.java | 137 ++---------------- .../iceberg/IcebergCatalogHandler.java | 113 +++------------ .../persistence/MetadataCacheManager.java | 38 ++--- 4 files changed, 69 insertions(+), 297 deletions(-) rename {polaris-service => service/common}/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java (87%) diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java index 11caac5be2..c7f4d96f7c 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java @@ -59,11 +59,8 @@ import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableMetadataParser; -<<<<<<< HEAD:polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java import org.apache.iceberg.TableOperations; -======= import org.apache.iceberg.UpdateSchema; ->>>>>>> cfe22b7de57bfeb6f877bee54b8a959f30173451:quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java import org.apache.iceberg.catalog.CatalogTests; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; @@ -96,20 +93,16 @@ import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -<<<<<<< HEAD:polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; -import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView; -======= import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.core.persistence.dao.entity.BaseResult; import org.apache.polaris.core.persistence.dao.entity.EntityResult; import org.apache.polaris.core.persistence.dao.entity.PrincipalSecretsResult; +import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView; import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.secrets.UserSecretsManager; import org.apache.polaris.core.secrets.UserSecretsManagerFactory; ->>>>>>> cfe22b7de57bfeb6f877bee54b8a959f30173451:quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.core.storage.PolarisStorageIntegration; @@ -127,12 +120,8 @@ import org.apache.polaris.service.config.RealmEntityManagerFactory; import org.apache.polaris.service.exception.FakeAzureHttpResponse; import org.apache.polaris.service.exception.IcebergExceptionMapper; -<<<<<<< HEAD:polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java -import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; import org.apache.polaris.service.persistence.MetadataCacheManager; -======= import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; ->>>>>>> cfe22b7de57bfeb6f877bee54b8a959f30173451:quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java import org.apache.polaris.service.task.TableCleanupTaskHandler; import org.apache.polaris.service.task.TaskExecutor; import org.apache.polaris.service.task.TaskFileIOSupplier; @@ -223,9 +212,7 @@ public Map getConfigOverrides() { private PolarisEntityManager entityManager; private FileIOFactory fileIOFactory; private PolarisEntity catalogEntity; -<<<<<<< HEAD:polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java private PolarisResolutionManifestCatalogView passthroughView; -======= private SecurityContext securityContext; @BeforeAll @@ -237,7 +224,6 @@ public static void setUpMocks() { @Nullable protected abstract EntityCache createEntityCache(PolarisMetaStoreManager metaStoreManager); ->>>>>>> cfe22b7de57bfeb6f877bee54b8a959f30173451:quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java @BeforeEach @SuppressWarnings("unchecked") @@ -298,52 +284,20 @@ public void before(TestInfo testInfo) { .setAllowedLocations(List.of(storageLocation, "s3://externally-owned-bucket")) .build(); catalogEntity = - adminService.createCatalog( -<<<<<<< HEAD:polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java - new CatalogEntity.Builder() - .setName(CATALOG_NAME) - .setDefaultBaseLocation(storageLocation) - .setReplaceNewLocationPrefixWithCatalogDefault("file:") - .addProperty( - PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") - .addProperty( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .addProperty(PolarisConfiguration.METADATA_CACHE_MAX_BYTES.catalogConfig(), "-1") - .setStorageConfigurationInfo(storageConfigModel, storageLocation) - .build()); + new CatalogEntity.Builder() + .setName(CATALOG_NAME) + .setDefaultBaseLocation(storageLocation) + .setReplaceNewLocationPrefixWithCatalogDefault("file:") + .addProperty(FeatureConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") + .addProperty( + FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") + .addProperty(FeatureConfiguration.METADATA_CACHE_MAX_BYTES.catalogConfig(), "-1") + .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .build(); passthroughView = new PolarisPassthroughResolutionView( - callContext, entityManager, authenticatedRoot, CATALOG_NAME); - TaskExecutor taskExecutor = Mockito.mock(); - this.catalog = - new BasePolarisCatalog( - entityManager, - metaStoreManager, - callContext, - passthroughView, - authenticatedRoot, - taskExecutor, - new DefaultFileIOFactory()); - this.catalog.initialize( - CATALOG_NAME, - ImmutableMap.of( - CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); - stsClient = Mockito.mock(StsClient.class); -======= - new CreateCatalogRequest( - new CatalogEntity.Builder() - .setName(CATALOG_NAME) - .setDefaultBaseLocation(storageLocation) - .setReplaceNewLocationPrefixWithCatalogDefault("file:") - .addProperty( - FeatureConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") - .addProperty( - FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), - "true") - .setStorageConfigurationInfo(storageConfigModel, storageLocation) - .build() - .asCatalog())); + callContext, entityManager, securityContext, CATALOG_NAME); RealmEntityManagerFactory realmEntityManagerFactory = new RealmEntityManagerFactory(createMockMetaStoreManagerFactory()); @@ -351,7 +305,6 @@ public void before(TestInfo testInfo) { new DefaultFileIOFactory(realmEntityManagerFactory, managerFactory, configurationStore); StsClient stsClient = Mockito.mock(StsClient.class); ->>>>>>> cfe22b7de57bfeb6f877bee54b8a959f30173451:quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java when(stsClient.assumeRole(isA(AssumeRoleRequest.class))) .thenReturn( AssumeRoleResponse.builder() @@ -1808,7 +1761,6 @@ public FileIO loadFileIO( Assertions.assertThat(measured.getNumDeletedFiles()).as("A table was deleted").isGreaterThan(0); } -<<<<<<< HEAD:polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java private Schema buildSchema(int fields) { Types.NestedField[] fieldsArray = new Types.NestedField[fields]; for (int i = 0; i < fields; i++) { @@ -1848,7 +1800,7 @@ public void testMetadataCachingWithManualFallback() { // Update the table TableOperations tableOps = catalog.newTableOps(tableIdentifier); - TableMetadata updatedMetadata = tableOps.current().updateSchema(buildSchema(100), 100); + TableMetadata updatedMetadata = tableOps.current().updateSchema(buildSchema(100)); tableOps.commit(tableOps.current(), updatedMetadata); // Read from the cache; it should detect a chance due to the update and load the new fallback @@ -1866,7 +1818,8 @@ public void testMetadataCachingWithManualFallback() { Assertions.assertThat(reloadedMetadata).isNotSameAs(cachedMetadata); Assertions.assertThat(reloadedMetadata.schema().columns().size()).isEqualTo(100); -======= + } + @Test public void testRegisterTableWithSlashlessMetadataLocation() { IcebergCatalog catalog = catalog(); @@ -2026,6 +1979,5 @@ public void testTableOperationsDoesNotRefreshAfterCommit(boolean updateMetadataO private static InMemoryFileIO getInMemoryIo(IcebergCatalog catalog) { return (InMemoryFileIO) ((ExceptionMappingFileIO) catalog.getIo()).getInnerIo(); ->>>>>>> cfe22b7de57bfeb6f877bee54b8a959f30173451:quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java } } diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index 1eba0d8777..d32c4ee357 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -25,14 +25,9 @@ import com.google.common.base.Objects; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; -<<<<<<< HEAD:polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java -import com.google.common.collect.Iterables; -import com.google.common.collect.Sets; -======= import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; import jakarta.ws.rs.core.SecurityContext; ->>>>>>> cfe22b7de57bfeb6f877bee54b8a959f30173451:service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java import java.io.Closeable; import java.io.IOException; import java.util.Arrays; @@ -63,10 +58,6 @@ import org.apache.iceberg.TableMetadataParser; import org.apache.iceberg.TableOperations; import org.apache.iceberg.TableProperties; -<<<<<<< HEAD:polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java -import org.apache.iceberg.aws.s3.S3FileIOProperties; -======= ->>>>>>> cfe22b7de57bfeb6f877bee54b8a959f30173451:service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.catalog.TableIdentifier; @@ -85,13 +76,9 @@ import org.apache.iceberg.io.CloseableGroup; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; -<<<<<<< HEAD:polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java -import org.apache.iceberg.io.SupportsBulkOperations; -======= import org.apache.iceberg.io.LocationProvider; import org.apache.iceberg.io.OutputFile; import org.apache.iceberg.util.LocationUtil; ->>>>>>> cfe22b7de57bfeb6f877bee54b8a959f30173451:service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java import org.apache.iceberg.util.PropertyUtil; import org.apache.iceberg.util.Tasks; import org.apache.iceberg.view.BaseMetastoreViewCatalog; @@ -135,12 +122,8 @@ import org.apache.polaris.core.storage.StorageLocation; import org.apache.polaris.service.catalog.SupportsNotifications; import org.apache.polaris.service.catalog.io.FileIOFactory; -<<<<<<< HEAD:polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java -import org.apache.polaris.service.exception.IcebergExceptionMapper; -import org.apache.polaris.service.persistence.MetadataCacheManager; -======= import org.apache.polaris.service.catalog.io.FileIOUtil; ->>>>>>> cfe22b7de57bfeb6f877bee54b8a959f30173451:service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +import org.apache.polaris.service.persistence.MetadataCacheManager; import org.apache.polaris.service.task.TaskExecutor; import org.apache.polaris.service.types.NotificationRequest; import org.apache.polaris.service.types.NotificationType; @@ -386,8 +369,9 @@ public TableOperations newTableOps( catalogFileIO, tableIdentifier, makeMetadataCurrentOnCommit); } + @VisibleForTesting @Override - protected TableOperations newTableOps(TableIdentifier tableIdentifier) { + public TableOperations newTableOps(TableIdentifier tableIdentifier) { boolean makeMetadataCurrentOnCommit = getCurrentPolarisContext() .getConfigurationStore() @@ -911,8 +895,8 @@ public TableMetadata loadTableMetadata(TableIdentifier identifier) { .getPolarisCallContext() .getConfigurationStore() .getConfiguration( - callContext.getPolarisCallContext(), PolarisConfiguration.METADATA_CACHE_MAX_BYTES); - if (maxMetadataCacheBytes == PolarisConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING) { + callContext.getPolarisCallContext(), FeatureConfiguration.METADATA_CACHE_MAX_BYTES); + if (maxMetadataCacheBytes == FeatureConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING) { return loadTableMetadata(loadTable(identifier)); } else { Supplier fallback = () -> loadTableMetadata(loadTable(identifier)); @@ -1317,18 +1301,21 @@ public TableMetadata refresh() { @Override public void commit(TableMetadata base, TableMetadata metadata) { - // if the metadata is already out of date, reject it - if (base != current()) { - if (base != null) { - throw new CommitFailedException("Cannot commit: stale table metadata"); - } else { + if (base == null) { + if (currentMetadata != null) { // when current is non-null, the table exists. but when base is null, the commit is trying // to create the table throw new AlreadyExistsException("Table already exists: %s", fullTableName); } - } - // if the metadata is not changed, return early - if (base == metadata) { + } else if (base.metadataFileLocation() != null + && !base.metadataFileLocation().equals(currentMetadata.metadataFileLocation())) { + throw new CommitFailedException("Cannot commit: stale table metadata"); + } else if (base != currentMetadata) { + // This branch is different from BaseMetastoreTableOperations + LOGGER.debug( + "Base object differs from current metadata; proceeding because locations match"); + } else if (base.metadataFileLocation().equals(metadata.metadataFileLocation())) { + // if the metadata is not changed, return early LOGGER.info("Nothing to commit."); return; } @@ -1588,98 +1575,6 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { } } - /** - * Copied from {@link BaseMetastoreTableOperations} but without the requirement that base == - * current() - * - * @param base table metadata on which changes were based - * @param metadata new table metadata with updates - */ - @Override - public void commit(TableMetadata base, TableMetadata metadata) { - TableMetadata currentMetadata = current(); - - // if the metadata is already out of date, reject it - if (base == null) { - if (currentMetadata != null) { - // when current is non-null, the table exists. but when base is null, the commit is trying - // to create the table - throw new AlreadyExistsException("Table already exists: %s", tableName()); - } - } else if (base.metadataFileLocation() != null - && !base.metadataFileLocation().equals(currentMetadata.metadataFileLocation())) { - throw new CommitFailedException("Cannot commit: stale table metadata"); - } else if (base != currentMetadata) { - // This branch is different from BaseMetastoreTableOperations - LOGGER.debug( - "Base object differs from current metadata; proceeding because locations match"); - } else if (base.metadataFileLocation().equals(metadata.metadataFileLocation())) { - // if the metadata is not changed, return early - LOGGER.info("Nothing to commit."); - return; - } - - long start = System.currentTimeMillis(); - doCommit(base, metadata); - deleteRemovedMetadataFiles(base, metadata); - requestRefresh(); - - LOGGER.info( - "Successfully committed to table {} in {} ms", - tableName(), - System.currentTimeMillis() - start); - } - - /** - * Copied from {@link BaseMetastoreTableOperations} as the method is private there This is moved - * to `CatalogUtils` in Iceberg 1.7.0 and can be called from there once we depend on Iceberg - * 1.7.0 - * - *

Deletes the oldest metadata files if {@link - * TableProperties#METADATA_DELETE_AFTER_COMMIT_ENABLED} is true. - * - * @param base table metadata on which previous versions were based - * @param metadata new table metadata with updated previous versions - */ - protected void deleteRemovedMetadataFiles(TableMetadata base, TableMetadata metadata) { - if (base == null) { - return; - } - - boolean deleteAfterCommit = - metadata.propertyAsBoolean( - TableProperties.METADATA_DELETE_AFTER_COMMIT_ENABLED, - TableProperties.METADATA_DELETE_AFTER_COMMIT_ENABLED_DEFAULT); - - if (deleteAfterCommit) { - Set removedPreviousMetadataFiles = - Sets.newHashSet(base.previousFiles()); - // TableMetadata#addPreviousFile builds up the metadata log and uses - // TableProperties.METADATA_PREVIOUS_VERSIONS_MAX to determine how many files should stay in - // the log, thus we don't include metadata.previousFiles() for deletion - everything else - // can - // be removed - removedPreviousMetadataFiles.removeAll(metadata.previousFiles()); - if (io() instanceof SupportsBulkOperations) { - ((SupportsBulkOperations) io()) - .deleteFiles( - Iterables.transform( - removedPreviousMetadataFiles, TableMetadata.MetadataLogEntry::file)); - } else { - Tasks.foreach(removedPreviousMetadataFiles) - .noRetry() - .suppressFailureWhenFinished() - .onFailure( - (previousMetadataFile, exc) -> - LOGGER.warn( - "Delete failed for previous metadata file: {}", - previousMetadataFile, - exc)) - .run(previousMetadataFile -> io().deleteFile(previousMetadataFile.file())); - } - } - } - @Override public TableOperations temp(TableMetadata uncommittedMetadata) { return new TableOperations() { diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index 482cc663fe..0054dab038 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -376,42 +376,13 @@ public LoadTableResponse createTableDirectWithWriteDelegation( .withSortOrder(request.writeOrder()) .withProperties(properties) .create(); - -<<<<<<< HEAD:polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java - if (table instanceof BaseTable baseTable) { - - final TableMetadata tableMetadata; - if (baseCatalog instanceof BasePolarisCatalog basePolarisCatalog) { - tableMetadata = basePolarisCatalog.loadTableMetadata(tableIdentifier); - } else { - tableMetadata = baseTable.operations().current(); - } - - LoadTableResponse.Builder responseBuilder = - LoadTableResponse.builder().withTableMetadata(tableMetadata); - if (baseCatalog instanceof SupportsCredentialDelegation credentialDelegation) { - LOGGER - .atDebug() - .addKeyValue("tableIdentifier", tableIdentifier) - .addKeyValue("tableLocation", tableMetadata.location()) - .log("Fetching client credentials for table"); - responseBuilder.addAllConfig( - credentialDelegation.getCredentialConfig( - tableIdentifier, - tableMetadata, - Set.of( - PolarisStorageActions.READ, - PolarisStorageActions.WRITE, - PolarisStorageActions.LIST))); - } - return responseBuilder.build(); - } else if (table instanceof BaseMetadataTable) { - // metadata tables are loaded on the client side, return NoSuchTableException for now - throw new NoSuchTableException("Table does not exist: %s", tableIdentifier.toString()); - } -======= if (table instanceof BaseTable baseTable) { - TableMetadata tableMetadata = baseTable.operations().current(); + final TableMetadata tableMetadata; + if (baseCatalog instanceof IcebergCatalog icebergCatalog) { + tableMetadata = icebergCatalog.loadTableMetadata(tableIdentifier); + } else { + tableMetadata = baseTable.operations().current(); + } return buildLoadTableResponseWithDelegationCredentials( tableIdentifier, tableMetadata, @@ -425,8 +396,6 @@ public LoadTableResponse createTableDirectWithWriteDelegation( // metadata tables are loaded on the client side, return NoSuchTableException for now throw new NoSuchTableException("Table does not exist: %s", tableIdentifier.toString()); } ->>>>>>> cfe22b7de57bfeb6f877bee54b8a959f30173451:service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java - throw new IllegalStateException("Cannot wrap catalog that does not produce BaseTable"); } @@ -576,19 +545,6 @@ public boolean sendNotification(TableIdentifier identifier, NotificationRequest */ private IcebergTableLikeEntity getTableEntity(TableIdentifier tableIdentifier) { PolarisResolvedPathWrapper target = resolutionManifest.getResolvedPath(tableIdentifier); - -<<<<<<< HEAD:polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java - return doCatalogOperation( - () -> { - if (baseCatalog instanceof BasePolarisCatalog basePolarisCatalog) { - return LoadTableResponse.builder() - .withTableMetadata(basePolarisCatalog.loadTableMetadata(tableIdentifier)) - .build(); - } - - return CatalogHandlers.loadTable(baseCatalog, tableIdentifier); - }); -======= return IcebergTableLikeEntity.of(target.getRawLeafEntity()); } @@ -634,7 +590,6 @@ public Optional loadTableIfStale( LoadTableResponse rawResponse = CatalogHandlers.loadTable(baseCatalog, tableIdentifier); return Optional.of(filterResponseToSnapshots(rawResponse, snapshots)); ->>>>>>> cfe22b7de57bfeb6f877bee54b8a959f30173451:service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java } public LoadTableResponse loadTableWithAccessDelegation( @@ -724,53 +679,20 @@ public Optional loadTableWithAccessDelegationIfStale( } } } + TableMetadata tableMetadata = null; + if (baseCatalog instanceof IcebergCatalog icebergCatalog) { + tableMetadata = icebergCatalog.loadTableMetadata(tableIdentifier); + } - // TODO: Find a way for the configuration or caller to better express whether to fail or omit - // when data-access is specified but access delegation grants are not found. -<<<<<<< HEAD:polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java - return doCatalogOperation( - () -> { - TableMetadata tableMetadata = null; - if (baseCatalog instanceof BasePolarisCatalog basePolarisCatalog) { - tableMetadata = basePolarisCatalog.loadTableMetadata(tableIdentifier); - } - - // The metadata failed to load - if (tableMetadata == null) { - throw new NoSuchTableException("Table does not exist: %s", tableIdentifier.toString()); - } - - if (baseCatalog instanceof SupportsCredentialDelegation credentialDelegation) { - LoadTableResponse.Builder responseBuilder = - LoadTableResponse.builder().withTableMetadata(tableMetadata); - LOGGER - .atDebug() - .addKeyValue("tableIdentifier", tableIdentifier) - .addKeyValue("tableLocation", tableMetadata.location()) - .log("Fetching client credentials for table"); - responseBuilder.addAllConfig( - credentialDelegation.getCredentialConfig( - tableIdentifier, tableMetadata, actionsRequested)); - return responseBuilder.build(); - } else { - throw new IllegalStateException("Cannot wrap catalog that does not produce BaseTable"); - } - }); -======= - Table table = baseCatalog.loadTable(tableIdentifier); - - if (table instanceof BaseTable baseTable) { - TableMetadata tableMetadata = baseTable.operations().current(); + // The metadata failed to load + if (tableMetadata == null) { + throw new NoSuchTableException("Table does not exist: %s", tableIdentifier.toString()); + } else { return Optional.of( buildLoadTableResponseWithDelegationCredentials( - tableIdentifier, tableMetadata, actionsRequested, snapshots) + tableIdentifier, tableMetadata, actionsRequested, snapshots) .build()); - } else if (table instanceof BaseMetadataTable) { - // metadata tables are loaded on the client side, return NoSuchTableException for now - throw new NoSuchTableException("Table does not exist: %s", tableIdentifier.toString()); } - - throw new IllegalStateException("Cannot wrap catalog that does not produce BaseTable"); } private LoadTableResponse.Builder buildLoadTableResponseWithDelegationCredentials( @@ -798,7 +720,6 @@ private LoadTableResponse.Builder buildLoadTableResponseWithDelegationCredential } } return responseBuilder; ->>>>>>> cfe22b7de57bfeb6f877bee54b8a959f30173451:service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java } private UpdateTableRequest applyUpdateFilters(UpdateTableRequest request) { @@ -958,8 +879,8 @@ public void commitTransaction(CommitTransactionRequest commitTransactionRequest) TableOperations tableOps = ((BaseTable) table).operations(); final TableMetadata currentMetadata; - if (baseCatalog instanceof BasePolarisCatalog basePolarisCatalog) { - currentMetadata = basePolarisCatalog.loadTableMetadata(change.identifier()); + if (baseCatalog instanceof IcebergCatalog icebergCatalog) { + currentMetadata = icebergCatalog.loadTableMetadata(change.identifier()); } else { currentMetadata = tableOps.current(); } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java similarity index 87% rename from polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java rename to service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 788f92f4e8..96d21e8478 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -31,12 +31,15 @@ import org.apache.iceberg.exceptions.RuntimeIOException; import org.apache.iceberg.util.JsonUtil; import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.PolarisConfiguration; +import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntitySubType; -import org.apache.polaris.core.entity.TableLikeEntity; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.table.IcebergTableLikeEntity; +import org.apache.polaris.core.entity.table.TableLikeEntity; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; +import org.apache.polaris.core.persistence.dao.entity.EntityResult; import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,15 +60,17 @@ public static TableMetadata loadTableMetadata( Supplier fallback) { LOGGER.debug(String.format("Loading cached metadata for %s", tableIdentifier)); PolarisResolvedPathWrapper resolvedEntities = - resolvedEntityView.getResolvedPath(tableIdentifier, PolarisEntitySubType.TABLE); - TableLikeEntity tableLikeEntity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); + resolvedEntityView.getResolvedPath( + tableIdentifier, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ICEBERG_TABLE); + IcebergTableLikeEntity tableLikeEntity = + IcebergTableLikeEntity.of(resolvedEntities.getRawLeafEntity()); String cacheContent = tableLikeEntity.getMetadataCacheContent(); if (cacheContent != null) { LOGGER.debug(String.format("Using cached metadata for %s", tableIdentifier)); return TableMetadataParser.fromJson(tableLikeEntity.getMetadataCacheContent()); } else { TableMetadata metadata = fallback.get(); - PolarisMetaStoreManager.EntityResult cacheResult = + var cacheResult = cacheTableMetadata( tableLikeEntity, metadata, @@ -103,8 +108,8 @@ public static Optional toBoundedJson(TableMetadata metadata, int maxByte * * @return The result of trying to cache the metadata */ - private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( - TableLikeEntity tableLikeEntity, + private static EntityResult cacheTableMetadata( + IcebergTableLikeEntity tableLikeEntity, TableMetadata metadata, int maxBytesToCache, PolarisCallContext callContext, @@ -112,24 +117,25 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( PolarisResolutionManifestCatalogView resolvedEntityView) { Optional jsonOpt = toBoundedJson(metadata, maxBytesToCache); // We should not reach this method in this case, but check just in case... - if (maxBytesToCache != PolarisConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING) { + if (maxBytesToCache != FeatureConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING) { if (jsonOpt.isEmpty()) { LOGGER.debug( String.format( "Will not cache metadata for %s; metadata above the limit of %d bytes", tableLikeEntity.getTableIdentifier(), maxBytesToCache)); - return new PolarisMetaStoreManager.EntityResult( - PolarisMetaStoreManager.EntityResult.ReturnStatus.SUCCESS, null); + return new EntityResult(EntityResult.ReturnStatus.SUCCESS, null); } else { LOGGER.debug( String.format("Caching metadata for %s", tableLikeEntity.getTableIdentifier())); TableLikeEntity newTableLikeEntity = - new TableLikeEntity.Builder(tableLikeEntity) + new IcebergTableLikeEntity.Builder(tableLikeEntity) .setMetadataContent(tableLikeEntity.getMetadataLocation(), jsonOpt.get()) .build(); PolarisResolvedPathWrapper resolvedPath = resolvedEntityView.getResolvedPath( - tableLikeEntity.getTableIdentifier(), PolarisEntitySubType.TABLE); + tableLikeEntity.getTableIdentifier(), + PolarisEntityType.TABLE_LIKE, + PolarisEntitySubType.ICEBERG_TABLE); try { return metaStoreManager.updateEntityPropertiesIfNotChanged( callContext, @@ -143,9 +149,8 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( String.format( "Encountered an error while caching %s: %s", tableLikeEntity.getTableIdentifier(), e)); - return new PolarisMetaStoreManager.EntityResult( - PolarisMetaStoreManager.EntityResult.ReturnStatus.UNEXPECTED_ERROR_SIGNALED, - e.getMessage()); + return new EntityResult( + EntityResult.ReturnStatus.UNEXPECTED_ERROR_SIGNALED, e.getMessage()); } else { throw e; } @@ -156,8 +161,7 @@ private static PolarisMetaStoreManager.EntityResult cacheTableMetadata( String.format( "Will not cache metadata for %s; metadata caching is disabled", tableLikeEntity.getTableIdentifier())); - return new PolarisMetaStoreManager.EntityResult( - PolarisMetaStoreManager.EntityResult.ReturnStatus.SUCCESS, null); + return new EntityResult(EntityResult.ReturnStatus.SUCCESS, null); } } From 3cd4b6a1e46ed108f52b5d4f417844bac3f9eae7 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Tue, 29 Apr 2025 10:31:29 -0700 Subject: [PATCH 59/88] autolint --- .../core/config/FeatureConfiguration.java | 1 - .../core/config/PolarisConfiguration.java | 4 +-- .../iceberg/IcebergCatalogHandler.java | 27 +++++++++++-------- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java index 776970d500..04452fe4f7 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java @@ -21,7 +21,6 @@ import java.util.List; import java.util.Optional; import java.util.function.Function; - import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.persistence.cache.EntityWeigher; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/PolarisConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/PolarisConfiguration.java index 6f89fa7f0a..42111ae413 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/PolarisConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/PolarisConfiguration.java @@ -22,7 +22,6 @@ import java.util.List; import java.util.Optional; import java.util.function.Function; - import org.apache.polaris.core.context.CallContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -141,7 +140,8 @@ public BehaviorChangeConfiguration buildBehaviorChangeConfiguration() { throw new IllegalArgumentException( "catalogConfig is not valid for behavior change configs"); } - return new BehaviorChangeConfiguration<>(key, description, defaultValue, catalogConfig, validation); + return new BehaviorChangeConfiguration<>( + key, description, defaultValue, catalogConfig, validation); } } diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index 0054dab038..3bdd21ded0 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -690,7 +690,7 @@ public Optional loadTableWithAccessDelegationIfStale( } else { return Optional.of( buildLoadTableResponseWithDelegationCredentials( - tableIdentifier, tableMetadata, actionsRequested, snapshots) + tableIdentifier, tableMetadata, actionsRequested, snapshots) .build()); } } @@ -811,8 +811,11 @@ public void tableExists(TableIdentifier tableIdentifier) { authorizeBasicTableLikeOperationOrThrow( op, PolarisEntitySubType.ICEBERG_TABLE, tableIdentifier); - // TODO: Just skip CatalogHandlers for this one maybe - CatalogHandlers.loadTable(baseCatalog, tableIdentifier); + if (baseCatalog instanceof IcebergCatalog icebergCatalog) { + icebergCatalog.loadTableMetadata(tableIdentifier); + } else { + CatalogHandlers.loadTable(baseCatalog, tableIdentifier); + } } public void renameTable(RenameTableRequest request) { @@ -870,18 +873,20 @@ public void commitTransaction(CommitTransactionRequest commitTransactionRequest) commitTransactionRequest.tableChanges().stream() .forEach( change -> { - final Table table = baseCatalog.loadTable(change.identifier()); - - if (!(table instanceof BaseTable)) { - throw new IllegalStateException( - "Cannot wrap catalog that does not produce BaseTable"); - } - - TableOperations tableOps = ((BaseTable) table).operations(); final TableMetadata currentMetadata; + final TableOperations tableOps; if (baseCatalog instanceof IcebergCatalog icebergCatalog) { + tableOps = icebergCatalog.newTableOps(change.identifier()); currentMetadata = icebergCatalog.loadTableMetadata(change.identifier()); } else { + final Table table = baseCatalog.loadTable(change.identifier()); + + if (!(table instanceof BaseTable)) { + throw new IllegalStateException( + "Cannot wrap catalog that does not produce BaseTable"); + } + + tableOps = ((BaseTable) table).operations(); currentMetadata = tableOps.current(); } From 5d7d16941c32050d1f5e6d38a901dcdbb87a8750 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Tue, 29 Apr 2025 10:49:32 -0700 Subject: [PATCH 60/88] stable again --- .../polaris/service/quarkus/catalog/IcebergCatalogTest.java | 6 +++++- .../polaris/service/catalog/iceberg/IcebergCatalog.java | 1 + .../service/catalog/iceberg/IcebergCatalogHandler.java | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java index c7f4d96f7c..bde1591bbe 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java @@ -283,7 +283,7 @@ public void before(TestInfo testInfo) { .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) .setAllowedLocations(List.of(storageLocation, "s3://externally-owned-bucket")) .build(); - catalogEntity = + CatalogEntity catalogEntityWithProperties = new CatalogEntity.Builder() .setName(CATALOG_NAME) .setDefaultBaseLocation(storageLocation) @@ -295,6 +295,10 @@ public void before(TestInfo testInfo) { .setStorageConfigurationInfo(storageConfigModel, storageLocation) .build(); + catalogEntity = + adminService.createCatalog( + new CreateCatalogRequest(catalogEntityWithProperties.asCatalog())); + passthroughView = new PolarisPassthroughResolutionView( callContext, entityManager, securityContext, CATALOG_NAME); diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index d32c4ee357..10704e15b5 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -1308,6 +1308,7 @@ public void commit(TableMetadata base, TableMetadata metadata) { throw new AlreadyExistsException("Table already exists: %s", fullTableName); } } else if (base.metadataFileLocation() != null + && currentMetadata != null && !base.metadataFileLocation().equals(currentMetadata.metadataFileLocation())) { throw new CommitFailedException("Cannot commit: stale table metadata"); } else if (base != currentMetadata) { diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index 3bdd21ded0..6e4ea6e31c 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -876,8 +876,8 @@ public void commitTransaction(CommitTransactionRequest commitTransactionRequest) final TableMetadata currentMetadata; final TableOperations tableOps; if (baseCatalog instanceof IcebergCatalog icebergCatalog) { - tableOps = icebergCatalog.newTableOps(change.identifier()); currentMetadata = icebergCatalog.loadTableMetadata(change.identifier()); + tableOps = icebergCatalog.newTableOps(change.identifier()); } else { final Table table = baseCatalog.loadTable(change.identifier()); From f6afa2e0f4bb3dca6c7ddcfcbe0560c84a8cf272 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 30 Apr 2025 13:26:21 -0700 Subject: [PATCH 61/88] missing callsite --- .../catalog/iceberg/IcebergCatalogHandler.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index 6e4ea6e31c..d1704bb681 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -588,7 +588,16 @@ public Optional loadTableIfStale( } } - LoadTableResponse rawResponse = CatalogHandlers.loadTable(baseCatalog, tableIdentifier); + final LoadTableResponse rawResponse; + if (baseCatalog instanceof IcebergCatalog icebergCatalog) { + TableMetadata tableMetadata = icebergCatalog.loadTableMetadata(tableIdentifier); + rawResponse = LoadTableResponse + .builder() + .withTableMetadata(tableMetadata) + .build(); + } else { + rawResponse = CatalogHandlers.loadTable(baseCatalog, tableIdentifier); + } return Optional.of(filterResponseToSnapshots(rawResponse, snapshots)); } From 8739bfe622bb9f88fc21ec140ff7187aa0c92384 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 30 Apr 2025 13:26:24 -0700 Subject: [PATCH 62/88] autolint --- .../service/catalog/iceberg/IcebergCatalogHandler.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index d1704bb681..538cae4828 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -591,10 +591,7 @@ public Optional loadTableIfStale( final LoadTableResponse rawResponse; if (baseCatalog instanceof IcebergCatalog icebergCatalog) { TableMetadata tableMetadata = icebergCatalog.loadTableMetadata(tableIdentifier); - rawResponse = LoadTableResponse - .builder() - .withTableMetadata(tableMetadata) - .build(); + rawResponse = LoadTableResponse.builder().withTableMetadata(tableMetadata).build(); } else { rawResponse = CatalogHandlers.loadTable(baseCatalog, tableIdentifier); } From 4f19dac71f4addbfb94a55cdfff3f016ad8608b8 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 30 Apr 2025 13:58:53 -0700 Subject: [PATCH 63/88] fixes --- .../service/quarkus/catalog/io/FileIOExceptionsTest.java | 1 + .../polaris/service/persistence/MetadataCacheManager.java | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/io/FileIOExceptionsTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/io/FileIOExceptionsTest.java index 9302daee89..4ae234e55a 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/io/FileIOExceptionsTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/io/FileIOExceptionsTest.java @@ -136,6 +136,7 @@ private static void requestLoadTable() { "ns1", "t1", null, + null, "ALL", services.realmContext(), services.securityContext()); diff --git a/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 96d21e8478..63ea478fde 100644 --- a/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -44,6 +44,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Contains utility methods related to storing TableMetadata in the metastore + * and retrieving it from the metastore + */ public class MetadataCacheManager { private static final Logger LOGGER = LoggerFactory.getLogger(MetadataCacheManager.class); @@ -181,7 +185,7 @@ public BoundedStringWriter(StringWriter delegate, int maxBytes) { } } - private boolean canWriteBytes(long bytesToWrite) throws IOException { + private boolean canWriteBytes(long bytesToWrite) { if (writtenBytes + bytesToWrite > maxBytes) { limitExceeded = true; } From 55dd50dcc1675d5eddb85800fd89c0d2d6bf9538 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 30 Apr 2025 13:58:56 -0700 Subject: [PATCH 64/88] autolint --- .../polaris/service/persistence/MetadataCacheManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 63ea478fde..8b4868e5cf 100644 --- a/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -45,8 +45,8 @@ import org.slf4j.LoggerFactory; /** - * Contains utility methods related to storing TableMetadata in the metastore - * and retrieving it from the metastore + * Contains utility methods related to storing TableMetadata in the metastore and retrieving it from + * the metastore */ public class MetadataCacheManager { private static final Logger LOGGER = LoggerFactory.getLogger(MetadataCacheManager.class); From 7858aa6429f310e1d41bd59628e0ad635e5cd050 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 30 Apr 2025 15:19:46 -0700 Subject: [PATCH 65/88] experiment with direct json streaming --- .../IcebergCatalogHandlerAuthzTest.java | 14 +- .../quarkus/catalog/IcebergCatalogTest.java | 22 +- .../catalog/iceberg/IcebergCatalog.java | 69 ++---- .../iceberg/IcebergCatalogAdapter.java | 40 ++-- .../iceberg/IcebergCatalogHandler.java | 151 ++++++------ .../catalog/iceberg/IcebergMetadataUtil.java | 55 +++++ .../iceberg/SupportsCredentialDelegation.java | 12 +- .../persistence/MetadataCacheManager.java | 220 +++++------------- .../service/persistence/MetadataJson.java | 43 ++++ 9 files changed, 324 insertions(+), 302 deletions(-) create mode 100644 service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergMetadataUtil.java create mode 100644 service/common/src/main/java/org/apache/polaris/service/persistence/MetadataJson.java diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java index 11a938f316..fdd0c67c5b 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java @@ -39,6 +39,7 @@ import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.ForbiddenException; import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.rest.RESTResponse; import org.apache.iceberg.rest.requests.CommitTransactionRequest; import org.apache.iceberg.rest.requests.CreateNamespaceRequest; import org.apache.iceberg.rest.requests.CreateTableRequest; @@ -48,6 +49,7 @@ import org.apache.iceberg.rest.requests.RenameTableRequest; import org.apache.iceberg.rest.requests.UpdateNamespacePropertiesRequest; import org.apache.iceberg.rest.requests.UpdateTableRequest; +import org.apache.iceberg.rest.responses.LoadTableResponse; import org.apache.iceberg.view.ImmutableSQLViewRepresentation; import org.apache.iceberg.view.ImmutableViewVersion; import org.apache.polaris.core.admin.model.CreateCatalogRequest; @@ -763,6 +765,14 @@ public void testCreateTableStagedWithWriteDelegationInsufficientPermissions() { }); } + private static String getMetadataLocation(RESTResponse resp) { + final String metadataLocation = + resp instanceof IcebergCatalogHandler.StringLoadTableResponse + ? ((IcebergCatalogHandler.StringLoadTableResponse) resp).metadataLocation() + : ((LoadTableResponse) resp).metadataLocation(); + return metadataLocation; + } + @Test public void testRegisterTableAllSufficientPrivileges() { Assertions.assertThat( @@ -776,7 +786,7 @@ public void testRegisterTableAllSufficientPrivileges() { // To get a handy metadata file we can use one from another table. // to avoid overlapping directories, drop the original table and recreate it via registerTable - final String metadataLocation = newWrapper().loadTable(TABLE_NS1_1, "all").metadataLocation(); + final String metadataLocation = getMetadataLocation(newWrapper().loadTable(TABLE_NS1_1, "all")); newWrapper(Set.of(PRINCIPAL_ROLE2)).dropTableWithoutPurge(TABLE_NS1_1); final RegisterTableRequest registerRequest = @@ -814,7 +824,7 @@ public void testRegisterTableInsufficientPermissions() { .isTrue(); // To get a handy metadata file we can use one from another table. - final String metadataLocation = newWrapper().loadTable(TABLE_NS1_1, "all").metadataLocation(); + final String metadataLocation = getMetadataLocation(newWrapper().loadTable(TABLE_NS1_1, "all")); final RegisterTableRequest registerRequest = new RegisterTableRequest() { diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java index fcf0c6c3bf..9485e25db0 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java @@ -121,6 +121,7 @@ import org.apache.polaris.service.exception.FakeAzureHttpResponse; import org.apache.polaris.service.exception.IcebergExceptionMapper; import org.apache.polaris.service.persistence.MetadataCacheManager; +import org.apache.polaris.service.persistence.MetadataJson; import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; import org.apache.polaris.service.task.TableCleanupTaskHandler; import org.apache.polaris.service.task.TaskExecutor; @@ -1785,8 +1786,8 @@ public void testMetadataCachingWithManualFallback() { Table createdTable = catalog.createTable(tableIdentifier, schema); TableMetadata originalMetadata = ((BaseTable) createdTable).operations().current(); - TableMetadata cachedMetadata = - MetadataCacheManager.loadTableMetadata( + MetadataJson cachedMetadataJson = + MetadataCacheManager.loadTableMetadataJson( tableIdentifier, Integer.MAX_VALUE, polarisContext, @@ -1796,11 +1797,8 @@ public void testMetadataCachingWithManualFallback() { throw new IllegalStateException("Fell back even though a cache entry should exist!"); }); - // The metadata object is loaded from the cache - Assertions.assertThat(cachedMetadata).isNotSameAs(originalMetadata); - // The content should match what was cached - Assertions.assertThat(TableMetadataParser.toJson(cachedMetadata)) + Assertions.assertThat(cachedMetadataJson.content()) .isEqualTo(TableMetadataParser.toJson(originalMetadata)); // Update the table @@ -1808,9 +1806,9 @@ public void testMetadataCachingWithManualFallback() { TableMetadata updatedMetadata = tableOps.current().updateSchema(buildSchema(100)); tableOps.commit(tableOps.current(), updatedMetadata); - // Read from the cache; it should detect a chance due to the update and load the new fallback - TableMetadata reloadedMetadata = - MetadataCacheManager.loadTableMetadata( + // Read from the cache; it should detect a change due to the update and load the new fallback + MetadataJson reloadedMetadataJson = + MetadataCacheManager.loadTableMetadataJson( tableIdentifier, Integer.MAX_VALUE, polarisContext, @@ -1821,8 +1819,10 @@ public void testMetadataCachingWithManualFallback() { "Fell back even though a cache entry should be updated on write"); }); - Assertions.assertThat(reloadedMetadata).isNotSameAs(cachedMetadata); - Assertions.assertThat(reloadedMetadata.schema().columns().size()).isEqualTo(100); + Assertions.assertThat(reloadedMetadataJson).isNotSameAs(cachedMetadataJson); + Assertions.assertThat( + TableMetadataParser.fromJson(reloadedMetadataJson.content()).schema().columns().size()) + .isEqualTo(100); } @Test diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index 2055c34c49..385c23c6b0 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -124,6 +124,7 @@ import org.apache.polaris.service.catalog.io.FileIOFactory; import org.apache.polaris.service.catalog.io.FileIOUtil; import org.apache.polaris.service.persistence.MetadataCacheManager; +import org.apache.polaris.service.persistence.MetadataJson; import org.apache.polaris.service.task.TaskExecutor; import org.apache.polaris.service.types.NotificationRequest; import org.apache.polaris.service.types.NotificationType; @@ -407,32 +408,6 @@ private String defaultNamespaceLocation(Namespace namespace) { } } - private Set getLocationsAllowedToBeAccessed(TableMetadata tableMetadata) { - Set locations = new HashSet<>(); - locations.add(tableMetadata.location()); - if (tableMetadata - .properties() - .containsKey(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY)) { - locations.add( - tableMetadata - .properties() - .get(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY)); - } - if (tableMetadata - .properties() - .containsKey(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY)) { - locations.add( - tableMetadata - .properties() - .get(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY)); - } - return locations; - } - - private Set getLocationsAllowedToBeAccessed(ViewMetadata viewMetadata) { - return Set.of(viewMetadata.location()); - } - @Override public boolean dropTable(TableIdentifier tableIdentifier, boolean purge) { TableOperations ops = newTableOps(tableIdentifier); @@ -868,7 +843,7 @@ public boolean sendNotification( @Override public Map getCredentialConfig( TableIdentifier tableIdentifier, - TableMetadata tableMetadata, + Set locationsAllowedToBeAccessed, Set storageActions) { Optional storageInfo = findStorageInfo(tableIdentifier); if (storageInfo.isEmpty()) { @@ -884,23 +859,26 @@ public Map getCredentialConfig( getCredentialVendor(), callContext.getPolarisCallContext().getConfigurationStore(), tableIdentifier, - getLocationsAllowedToBeAccessed(tableMetadata), + locationsAllowedToBeAccessed, storageActions, storageInfo.get()); } - public TableMetadata loadTableMetadata(TableIdentifier identifier) { + public MetadataJson loadTableMetadataJson(TableIdentifier identifier) { int maxMetadataCacheBytes = callContext .getPolarisCallContext() .getConfigurationStore() .getConfiguration( callContext.getPolarisCallContext(), FeatureConfiguration.METADATA_CACHE_MAX_BYTES); + Supplier fallback = + () -> { + return loadTableMetadata(loadTable(identifier)); + }; if (maxMetadataCacheBytes == FeatureConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING) { - return loadTableMetadata(loadTable(identifier)); + return MetadataJson.fromMetadata(fallback.get()); } else { - Supplier fallback = () -> loadTableMetadata(loadTable(identifier)); - return MetadataCacheManager.loadTableMetadata( + return MetadataCacheManager.loadTableMetadataJson( identifier, maxMetadataCacheBytes, callContext.getPolarisCallContext(), @@ -1401,7 +1379,7 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { tableFileIO = loadFileIOForTableLike( tableIdentifier, - getLocationsAllowedToBeAccessed(metadata), + IcebergMetadataUtil.getLocationsAllowedToBeAccessed(metadata), resolvedStorageEntity, new HashMap<>(metadata.properties()), Set.of(PolarisStorageActions.READ, PolarisStorageActions.WRITE)); @@ -1488,20 +1466,19 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { callContext.getPolarisCallContext(), catalogEntity, FeatureConfiguration.METADATA_CACHE_MAX_BYTES); - Optional metadataJsonOpt = Optional.empty(); - boolean shouldPersistMetadata = + Optional metadataJsonOpt = switch (maxMetadataCacheBytes) { - case FeatureConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING -> false; + case FeatureConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING -> Optional.empty(); case FeatureConfiguration.METADATA_CACHE_MAX_BYTES_INFINITE_CACHING -> { - metadataJsonOpt = MetadataCacheManager.toBoundedJson(metadata, maxMetadataCacheBytes); - yield true; + yield Optional.of(TableMetadataParser.toJson(metadata)); } default -> { - metadataJsonOpt = MetadataCacheManager.toBoundedJson(metadata, maxMetadataCacheBytes); - yield metadataJsonOpt - .map(String::length) - .map(l -> l <= maxMetadataCacheBytes) - .orElse(false); + metadataJsonOpt = Optional.of(TableMetadataParser.toJson(metadata)); + if (metadataJsonOpt.get().length() <= maxMetadataCacheBytes) { + yield Optional.of(TableMetadataParser.toJson(metadata)); + } else { + yield Optional.empty(); + } } }; final IcebergTableLikeEntity.Builder builder; @@ -1521,9 +1498,7 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { .setBaseLocation(metadata.location()) .setMetadataLocation(newLocation); } - if (shouldPersistMetadata && metadataJsonOpt.isPresent()) { - builder.setMetadataContent(newLocation, metadataJsonOpt.get()); - } + metadataJsonOpt.ifPresent(s -> builder.setMetadataContent(newLocation, s)); entity = builder.build(); if (!Objects.equal(existingLocation, oldLocation)) { if (null == base) { @@ -1818,7 +1793,7 @@ public void doCommit(ViewMetadata base, ViewMetadata metadata) { viewFileIO = loadFileIOForTableLike( identifier, - getLocationsAllowedToBeAccessed(metadata), + IcebergMetadataUtil.getLocationsAllowedToBeAccessed(metadata), resolvedStorageEntity, tableProperties, Set.of(PolarisStorageActions.READ, PolarisStorageActions.WRITE)); diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java index 5baffa3952..d27fdc5a9c 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java @@ -31,6 +31,7 @@ import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; import java.util.EnumSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -42,8 +43,10 @@ import org.apache.iceberg.exceptions.NotAuthorizedException; import org.apache.iceberg.exceptions.NotFoundException; import org.apache.iceberg.rest.Endpoint; +import org.apache.iceberg.rest.RESTResponse; import org.apache.iceberg.rest.RESTUtil; import org.apache.iceberg.rest.ResourcePaths; +import org.apache.iceberg.rest.credentials.Credential; import org.apache.iceberg.rest.requests.CommitTransactionRequest; import org.apache.iceberg.rest.requests.CreateNamespaceRequest; import org.apache.iceberg.rest.requests.CreateTableRequest; @@ -237,15 +240,21 @@ public Response loadNamespaceMetadata( * unable to get metadata location and logs a warning. */ private Response.ResponseBuilder tryInsertETagHeader( - Response.ResponseBuilder builder, - LoadTableResponse response, - String namespace, - String tableName) { - if (response.metadataLocation() != null) { + Response.ResponseBuilder builder, RESTResponse response, String namespace, String tableName) { + final String metadataLocation; + if (response instanceof LoadTableResponse loadTableResponse) { + metadataLocation = loadTableResponse.metadataLocation(); + } else if (response instanceof IcebergCatalogHandler.StringLoadTableResponse sltr) { + metadataLocation = sltr.metadataLocation(); + } else { + throw new IllegalStateException("Cannot build etag from: " + response); + } + + if (metadataLocation != null) { builder = builder.header( HttpHeaders.ETAG, - IcebergHttpUtil.generateETagForMetadataFileLocation(response.metadataLocation())); + IcebergHttpUtil.generateETagForMetadataFileLocation(metadataLocation)); } else { LOGGER .atWarn() @@ -337,7 +346,7 @@ public Response createTable( Response.ok(response), response, namespace, createTableRequest.name()) .build(); } else { - LoadTableResponse response = + RESTResponse response = catalog.createTableDirectWithWriteDelegation(ns, createTableRequest); return tryInsertETagHeader( Response.ok(response), response, namespace, createTableRequest.name()) @@ -384,7 +393,7 @@ public Response loadTable( securityContext, prefix, catalog -> { - LoadTableResponse response; + RESTResponse response; if (delegationModes.isEmpty()) { response = @@ -541,12 +550,17 @@ public Response loadCredentials( securityContext, prefix, catalog -> { - LoadTableResponse loadTableResponse = - catalog.loadTableWithAccessDelegation(tableIdentifier, "all"); + RESTResponse restResponse = catalog.loadTableWithAccessDelegation(tableIdentifier, "all"); + final List credentials; + if (restResponse instanceof LoadTableResponse loadTableResponse) { + credentials = loadTableResponse.credentials(); + } else if (restResponse instanceof IcebergCatalogHandler.StringLoadTableResponse sltr) { + credentials = sltr.credentials(); + } else { + throw new IllegalStateException("Cannot extract credentials from " + restResponse); + } return Response.ok( - ImmutableLoadCredentialsResponse.builder() - .credentials(loadTableResponse.credentials()) - .build()) + ImmutableLoadCredentialsResponse.builder().credentials(credentials).build()) .build(); }); } diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index 538cae4828..37db12c6d9 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.catalog.iceberg; +import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import jakarta.annotation.Nonnull; @@ -41,6 +42,7 @@ import org.apache.iceberg.SortOrder; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; import org.apache.iceberg.TableOperations; import org.apache.iceberg.UpdateRequirement; import org.apache.iceberg.catalog.Catalog; @@ -57,6 +59,8 @@ import org.apache.iceberg.rest.CatalogHandlers; import org.apache.iceberg.rest.HTTPClient; import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.rest.RESTResponse; +import org.apache.iceberg.rest.credentials.Credential; import org.apache.iceberg.rest.credentials.ImmutableCredential; import org.apache.iceberg.rest.requests.CommitTransactionRequest; import org.apache.iceberg.rest.requests.CreateNamespaceRequest; @@ -97,6 +101,7 @@ import org.apache.polaris.service.context.CallContextCatalogFactory; import org.apache.polaris.service.http.IcebergHttpUtil; import org.apache.polaris.service.http.IfNoneMatch; +import org.apache.polaris.service.persistence.MetadataJson; import org.apache.polaris.service.types.NotificationRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -341,7 +346,7 @@ public LoadTableResponse createTableDirect(Namespace namespace, CreateTableReque * @param request the table creation request * @return ETagged {@link LoadTableResponse} to uniquely identify the table metadata */ - public LoadTableResponse createTableDirectWithWriteDelegation( + public RESTResponse createTableDirectWithWriteDelegation( Namespace namespace, CreateTableRequest request) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.CREATE_TABLE_DIRECT_WITH_WRITE_DELEGATION; @@ -377,21 +382,18 @@ public LoadTableResponse createTableDirectWithWriteDelegation( .withProperties(properties) .create(); if (table instanceof BaseTable baseTable) { - final TableMetadata tableMetadata; + final MetadataJson tableMetadataJson; if (baseCatalog instanceof IcebergCatalog icebergCatalog) { - tableMetadata = icebergCatalog.loadTableMetadata(tableIdentifier); + tableMetadataJson = icebergCatalog.loadTableMetadataJson(tableIdentifier); } else { - tableMetadata = baseTable.operations().current(); + TableMetadata tableMetadata = baseTable.operations().current(); + tableMetadataJson = MetadataJson.fromMetadata(tableMetadata); } return buildLoadTableResponseWithDelegationCredentials( - tableIdentifier, - tableMetadata, - Set.of( - PolarisStorageActions.READ, - PolarisStorageActions.WRITE, - PolarisStorageActions.LIST), - SNAPSHOTS_ALL) - .build(); + tableIdentifier, + tableMetadataJson, + Set.of( + PolarisStorageActions.READ, PolarisStorageActions.WRITE, PolarisStorageActions.LIST)); } else if (table instanceof BaseMetadataTable) { // metadata tables are loaded on the client side, return NoSuchTableException for now throw new NoSuchTableException("Table does not exist: %s", tableIdentifier.toString()); @@ -460,7 +462,7 @@ public LoadTableResponse createTableStaged(Namespace namespace, CreateTableReque return LoadTableResponse.builder().withTableMetadata(metadata).build(); } - public LoadTableResponse createTableStagedWithWriteDelegation( + public RESTResponse createTableStagedWithWriteDelegation( Namespace namespace, CreateTableRequest request) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.CREATE_TABLE_STAGED_WITH_WRITE_DELEGATION; @@ -480,8 +482,7 @@ public LoadTableResponse createTableStagedWithWriteDelegation( TableMetadata metadata = stageTableCreateHelper(namespace, request); return buildLoadTableResponseWithDelegationCredentials( - ident, metadata, Set.of(PolarisStorageActions.ALL), SNAPSHOTS_ALL) - .build(); + ident, MetadataJson.fromMetadata(metadata), Set.of(PolarisStorageActions.ALL)); } /** @@ -548,7 +549,7 @@ private IcebergTableLikeEntity getTableEntity(TableIdentifier tableIdentifier) { return IcebergTableLikeEntity.of(target.getRawLeafEntity()); } - public LoadTableResponse loadTable(TableIdentifier tableIdentifier, String snapshots) { + public RESTResponse loadTable(TableIdentifier tableIdentifier, String snapshots) { return loadTableIfStale(tableIdentifier, null, snapshots).get(); } @@ -562,7 +563,7 @@ public LoadTableResponse loadTable(TableIdentifier tableIdentifier, String snaps * @return {@link Optional#empty()} if the ETag is current, an {@link Optional} containing the * load table response, otherwise */ - public Optional loadTableIfStale( + public Optional loadTableIfStale( TableIdentifier tableIdentifier, IfNoneMatch ifNoneMatch, String snapshots) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LOAD_TABLE; authorizeBasicTableLikeOperationOrThrow( @@ -588,17 +589,17 @@ public Optional loadTableIfStale( } } - final LoadTableResponse rawResponse; + final RESTResponse rawResponse; if (baseCatalog instanceof IcebergCatalog icebergCatalog) { - TableMetadata tableMetadata = icebergCatalog.loadTableMetadata(tableIdentifier); - rawResponse = LoadTableResponse.builder().withTableMetadata(tableMetadata).build(); + MetadataJson metadataJson = icebergCatalog.loadTableMetadataJson(tableIdentifier); + rawResponse = new StringLoadTableResponse(metadataJson); } else { rawResponse = CatalogHandlers.loadTable(baseCatalog, tableIdentifier); } return Optional.of(filterResponseToSnapshots(rawResponse, snapshots)); } - public LoadTableResponse loadTableWithAccessDelegation( + public RESTResponse loadTableWithAccessDelegation( TableIdentifier tableIdentifier, String snapshots) { return loadTableWithAccessDelegationIfStale(tableIdentifier, null, snapshots).get(); } @@ -613,7 +614,7 @@ public LoadTableResponse loadTableWithAccessDelegation( * @return {@link Optional#empty()} if the ETag is current, an {@link Optional} containing the * load table response, otherwise */ - public Optional loadTableWithAccessDelegationIfStale( + public Optional loadTableWithAccessDelegationIfStale( TableIdentifier tableIdentifier, IfNoneMatch ifNoneMatch, String snapshots) { // Here we have a single method that falls through multiple candidate // PolarisAuthorizableOperations because instead of identifying the desired operation up-front @@ -685,9 +686,9 @@ public Optional loadTableWithAccessDelegationIfStale( } } } - TableMetadata tableMetadata = null; + MetadataJson tableMetadata = null; if (baseCatalog instanceof IcebergCatalog icebergCatalog) { - tableMetadata = icebergCatalog.loadTableMetadata(tableIdentifier); + tableMetadata = icebergCatalog.loadTableMetadataJson(tableIdentifier); } // The metadata failed to load @@ -696,36 +697,32 @@ public Optional loadTableWithAccessDelegationIfStale( } else { return Optional.of( buildLoadTableResponseWithDelegationCredentials( - tableIdentifier, tableMetadata, actionsRequested, snapshots) - .build()); + tableIdentifier, tableMetadata, actionsRequested)); } } - private LoadTableResponse.Builder buildLoadTableResponseWithDelegationCredentials( + private RESTResponse buildLoadTableResponseWithDelegationCredentials( TableIdentifier tableIdentifier, - TableMetadata tableMetadata, - Set actions, - String snapshots) { - LoadTableResponse.Builder responseBuilder = - LoadTableResponse.builder().withTableMetadata(tableMetadata); + MetadataJson metadataJson, + Set actions) { + Map config = Map.of(); + List credentials = List.of(); if (baseCatalog instanceof SupportsCredentialDelegation credentialDelegation) { LOGGER .atDebug() .addKeyValue("tableIdentifier", tableIdentifier) - .addKeyValue("tableLocation", tableMetadata.location()) + .addKeyValue("tableLocation", metadataJson.location()) .log("Fetching client credentials for table"); - Map credentialConfig = - credentialDelegation.getCredentialConfig(tableIdentifier, tableMetadata, actions); - responseBuilder.addAllConfig(credentialConfig); - if (!credentialConfig.isEmpty()) { - responseBuilder.addCredential( - ImmutableCredential.builder() - .prefix(tableMetadata.location()) - .config(credentialConfig) - .build()); + config = + credentialDelegation.getCredentialConfig( + tableIdentifier, metadataJson.tableLocations(), actions); + if (!config.isEmpty()) { + credentials.add( + ImmutableCredential.builder().prefix(metadataJson.location()).config(config).build()); } } - return responseBuilder; + return new StringLoadTableResponse( + metadataJson.location(), metadataJson.content(), config, credentials); } private UpdateTableRequest applyUpdateFilters(UpdateTableRequest request) { @@ -818,7 +815,7 @@ public void tableExists(TableIdentifier tableIdentifier) { op, PolarisEntitySubType.ICEBERG_TABLE, tableIdentifier); if (baseCatalog instanceof IcebergCatalog icebergCatalog) { - icebergCatalog.loadTableMetadata(tableIdentifier); + icebergCatalog.loadTableMetadataJson(tableIdentifier); } else { CatalogHandlers.loadTable(baseCatalog, tableIdentifier); } @@ -882,7 +879,9 @@ public void commitTransaction(CommitTransactionRequest commitTransactionRequest) final TableMetadata currentMetadata; final TableOperations tableOps; if (baseCatalog instanceof IcebergCatalog icebergCatalog) { - currentMetadata = icebergCatalog.loadTableMetadata(change.identifier()); + currentMetadata = + TableMetadataParser.fromJson( + icebergCatalog.loadTableMetadataJson(change.identifier()).content()); tableOps = icebergCatalog.newTableOps(change.identifier()); } else { final Table table = baseCatalog.loadTable(change.identifier()); @@ -1035,28 +1034,32 @@ public void renameView(RenameTableRequest request) { CatalogHandlers.renameView(viewCatalog, request); } - private @Nonnull LoadTableResponse filterResponseToSnapshots( - LoadTableResponse loadTableResponse, String snapshots) { - if (snapshots == null || snapshots.equalsIgnoreCase(SNAPSHOTS_ALL)) { - return loadTableResponse; - } else if (snapshots.equalsIgnoreCase(SNAPSHOTS_REFS)) { - TableMetadata metadata = loadTableResponse.tableMetadata(); - - Set referencedSnapshotIds = - metadata.refs().values().stream() - .map(SnapshotRef::snapshotId) - .collect(Collectors.toSet()); - - TableMetadata filteredMetadata = - metadata.removeSnapshotsIf(s -> !referencedSnapshotIds.contains(s.snapshotId())); - - return LoadTableResponse.builder() - .withTableMetadata(filteredMetadata) - .addAllConfig(loadTableResponse.config()) - .addAllCredentials(loadTableResponse.credentials()) - .build(); + private @Nonnull RESTResponse filterResponseToSnapshots( + RESTResponse restResponse, String snapshots) { + if (restResponse instanceof LoadTableResponse loadTableResponse) { + if (snapshots == null || snapshots.equalsIgnoreCase(SNAPSHOTS_ALL)) { + return loadTableResponse; + } else if (snapshots.equalsIgnoreCase(SNAPSHOTS_REFS)) { + TableMetadata metadata = loadTableResponse.tableMetadata(); + + Set referencedSnapshotIds = + metadata.refs().values().stream() + .map(SnapshotRef::snapshotId) + .collect(Collectors.toSet()); + + TableMetadata filteredMetadata = + metadata.removeSnapshotsIf(s -> !referencedSnapshotIds.contains(s.snapshotId())); + + return LoadTableResponse.builder() + .withTableMetadata(filteredMetadata) + .addAllConfig(loadTableResponse.config()) + .addAllCredentials(loadTableResponse.credentials()) + .build(); + } else { + throw new IllegalArgumentException("Unrecognized snapshots: " + snapshots); + } } else { - throw new IllegalArgumentException("Unrecognized snapshots: " + snapshots); + return restResponse; } } @@ -1066,4 +1069,20 @@ public void close() throws Exception { closeable.close(); } } + + public record StringLoadTableResponse( + @JsonProperty("metadata-location") String metadataLocation, + @JsonProperty("metadata") String metadata, + @JsonProperty("config") Map config, + @JsonProperty("storage-credentials") List credentials) + implements RESTResponse { + @Override + public void validate() { + Preconditions.checkNotNull(this.metadata, "Invalid metadata: null"); + } + + public StringLoadTableResponse(MetadataJson metadataJson) { + this(metadataJson.location(), metadataJson.content(), Map.of(), List.of()); + } + } } diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergMetadataUtil.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergMetadataUtil.java new file mode 100644 index 0000000000..aefcf9d3a9 --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergMetadataUtil.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.iceberg; + +import java.util.HashSet; +import java.util.Set; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.view.ViewMetadata; +import org.apache.polaris.core.entity.table.IcebergTableLikeEntity; + +/** A collection of util methods related to handling Iceberg table & view metadata */ +public class IcebergMetadataUtil { + + public static Set getLocationsAllowedToBeAccessed(TableMetadata tableMetadata) { + Set locations = new HashSet<>(); + locations.add(tableMetadata.location()); + if (tableMetadata + .properties() + .containsKey(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY)) { + locations.add( + tableMetadata + .properties() + .get(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY)); + } + if (tableMetadata + .properties() + .containsKey(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY)) { + locations.add( + tableMetadata + .properties() + .get(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY)); + } + return locations; + } + + public static Set getLocationsAllowedToBeAccessed(ViewMetadata viewMetadata) { + return Set.of(viewMetadata.location()); + } +} diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/SupportsCredentialDelegation.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/SupportsCredentialDelegation.java index 06ca7fbde6..c9ff6620cd 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/SupportsCredentialDelegation.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/SupportsCredentialDelegation.java @@ -32,8 +32,18 @@ * configuration. */ public interface SupportsCredentialDelegation { - Map getCredentialConfig( + default Map getCredentialConfig( TableIdentifier tableIdentifier, TableMetadata tableMetadata, + Set storageActions) { + return getCredentialConfig( + tableIdentifier, + IcebergMetadataUtil.getLocationsAllowedToBeAccessed(tableMetadata), + storageActions); + } + + Map getCredentialConfig( + TableIdentifier tableIdentifier, + Set locationsAllowedToBeAccessed, Set storageActions); } diff --git a/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 8b4868e5cf..6811d4686c 100644 --- a/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -18,21 +18,17 @@ */ package org.apache.polaris.service.persistence; -import com.fasterxml.jackson.core.JsonGenerator; -import java.io.IOException; -import java.io.StringWriter; -import java.io.Writer; -import java.util.Optional; +import java.util.Map; +import java.util.Objects; import java.util.function.Supplier; -import org.apache.iceberg.Table; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.iceberg.TableMetadata; -import org.apache.iceberg.TableMetadataParser; import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.exceptions.RuntimeIOException; -import org.apache.iceberg.util.JsonUtil; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.table.IcebergTableLikeEntity; @@ -52,10 +48,10 @@ public class MetadataCacheManager { private static final Logger LOGGER = LoggerFactory.getLogger(MetadataCacheManager.class); /** - * Load the cached {@link Table} or fall back to `fallback` if one doesn't exist. If the metadata - * is not currently cached, it may be added to the cache. + * Load the cached metadata.json content and location or fall back to `fallback` if one doesn't + * exist. If the metadata is not currently cached, it may be added to the cache. */ - public static TableMetadata loadTableMetadata( + public static MetadataJson loadTableMetadataJson( TableIdentifier tableIdentifier, int maxBytesToCache, PolarisCallContext callContext, @@ -66,18 +62,33 @@ public static TableMetadata loadTableMetadata( PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getResolvedPath( tableIdentifier, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ICEBERG_TABLE); + // If the table doesn't exist, just fall back fast + if (resolvedEntities == null) { + return MetadataJson.fromMetadata(fallback.get()); + } IcebergTableLikeEntity tableLikeEntity = IcebergTableLikeEntity.of(resolvedEntities.getRawLeafEntity()); String cacheContent = tableLikeEntity.getMetadataCacheContent(); if (cacheContent != null) { LOGGER.debug(String.format("Using cached metadata for %s", tableIdentifier)); - return TableMetadataParser.fromJson(tableLikeEntity.getMetadataCacheContent()); + Map entityProperties = tableLikeEntity.getPropertiesAsMap(); + return new MetadataJson( + tableLikeEntity.getMetadataLocation(), + tableLikeEntity.getMetadataCacheContent(), + Stream.of( + entityProperties.get(PolarisEntityConstants.ENTITY_BASE_LOCATION), + entityProperties.get( + IcebergTableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY), + entityProperties.get( + IcebergTableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY)) + .filter(Objects::nonNull) + .collect(Collectors.toSet())); } else { - TableMetadata metadata = fallback.get(); + MetadataJson fallbackJson = MetadataJson.fromMetadata(fallback.get()); var cacheResult = - cacheTableMetadata( + cacheTableMetadataJson( tableLikeEntity, - metadata, + fallbackJson.content(), maxBytesToCache, callContext, metastoreManager, @@ -85,25 +96,7 @@ public static TableMetadata loadTableMetadata( if (!cacheResult.isSuccess()) { LOGGER.debug(String.format("Failed to cache metadata for %s", tableIdentifier)); } - return metadata; - } - } - - /** Convert a {@link TableMetadata} to JSON, with the size bounded */ - public static Optional toBoundedJson(TableMetadata metadata, int maxBytes) { - try (StringWriter unboundedWriter = new StringWriter()) { - BoundedStringWriter boundedWriter = new BoundedStringWriter(unboundedWriter, maxBytes); - JsonGenerator generator = JsonUtil.factory().createGenerator(boundedWriter); - TableMetadataParser.toJson(metadata, generator); - generator.flush(); - String result = boundedWriter.toString(); - if (boundedWriter.isLimitExceeded()) { - return Optional.empty(); - } else { - return Optional.ofNullable(result); - } - } catch (IOException e) { - throw new RuntimeIOException(e, "Failed to write json for: %s", metadata); + return fallbackJson; } } @@ -112,148 +105,51 @@ public static Optional toBoundedJson(TableMetadata metadata, int maxByte * * @return The result of trying to cache the metadata */ - private static EntityResult cacheTableMetadata( + private static EntityResult cacheTableMetadataJson( IcebergTableLikeEntity tableLikeEntity, - TableMetadata metadata, + String metadataJson, int maxBytesToCache, PolarisCallContext callContext, PolarisMetaStoreManager metaStoreManager, PolarisResolutionManifestCatalogView resolvedEntityView) { - Optional jsonOpt = toBoundedJson(metadata, maxBytesToCache); - // We should not reach this method in this case, but check just in case... - if (maxBytesToCache != FeatureConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING) { - if (jsonOpt.isEmpty()) { + if (maxBytesToCache != FeatureConfiguration.METADATA_CACHE_MAX_BYTES_INFINITE_CACHING) { + if (metadataJson.length() > maxBytesToCache) { LOGGER.debug( String.format( "Will not cache metadata for %s; metadata above the limit of %d bytes", tableLikeEntity.getTableIdentifier(), maxBytesToCache)); return new EntityResult(EntityResult.ReturnStatus.SUCCESS, null); - } else { - LOGGER.debug( - String.format("Caching metadata for %s", tableLikeEntity.getTableIdentifier())); - TableLikeEntity newTableLikeEntity = - new IcebergTableLikeEntity.Builder(tableLikeEntity) - .setMetadataContent(tableLikeEntity.getMetadataLocation(), jsonOpt.get()) - .build(); - PolarisResolvedPathWrapper resolvedPath = - resolvedEntityView.getResolvedPath( - tableLikeEntity.getTableIdentifier(), - PolarisEntityType.TABLE_LIKE, - PolarisEntitySubType.ICEBERG_TABLE); - try { - return metaStoreManager.updateEntityPropertiesIfNotChanged( - callContext, - PolarisEntity.toCoreList(resolvedPath.getRawParentPath()), - newTableLikeEntity); - } catch (RuntimeException e) { - // PersistenceException (& other extension-specific exceptions) may not be in scope, - // but we can make a best-effort attempt to swallow it and just forego caching - if (e.toString().contains("PersistenceException")) { - LOGGER.debug( - String.format( - "Encountered an error while caching %s: %s", - tableLikeEntity.getTableIdentifier(), e)); - return new EntityResult( - EntityResult.ReturnStatus.UNEXPECTED_ERROR_SIGNALED, e.getMessage()); - } else { - throw e; - } - } } - } else { - LOGGER.debug( - String.format( - "Will not cache metadata for %s; metadata caching is disabled", - tableLikeEntity.getTableIdentifier())); - return new EntityResult(EntityResult.ReturnStatus.SUCCESS, null); } - } - private static class BoundedStringWriter extends Writer { - private final Writer delegate; - private final int maxBytes; - private long writtenBytes = 0; - private boolean limitExceeded = false; - - /** Create a new BoundedWriter with a given limit `maxBytes`. -1 means no limit. */ - public BoundedStringWriter(StringWriter delegate, int maxBytes) { - this.delegate = delegate; - if (maxBytes == -1) { - this.maxBytes = Integer.MAX_VALUE; + LOGGER.debug(String.format("Caching metadata for %s", tableLikeEntity.getTableIdentifier())); + TableLikeEntity newTableLikeEntity = + new IcebergTableLikeEntity.Builder(tableLikeEntity) + .setMetadataContent(tableLikeEntity.getMetadataLocation(), metadataJson) + .build(); + PolarisResolvedPathWrapper resolvedPath = + resolvedEntityView.getResolvedPath( + tableLikeEntity.getTableIdentifier(), + PolarisEntityType.TABLE_LIKE, + PolarisEntitySubType.ICEBERG_TABLE); + try { + return metaStoreManager.updateEntityPropertiesIfNotChanged( + callContext, + PolarisEntity.toCoreList(resolvedPath.getRawParentPath()), + newTableLikeEntity); + } catch (RuntimeException e) { + // PersistenceException (& other extension-specific exceptions) may not be in scope, + // but we can make a best-effort attempt to swallow it and just forego caching + if (e.toString().contains("PersistenceException")) { + LOGGER.debug( + String.format( + "Encountered an error while caching %s: %s", + tableLikeEntity.getTableIdentifier(), e)); + return new EntityResult( + EntityResult.ReturnStatus.UNEXPECTED_ERROR_SIGNALED, e.getMessage()); } else { - this.maxBytes = maxBytes; - } - } - - private boolean canWriteBytes(long bytesToWrite) { - if (writtenBytes + bytesToWrite > maxBytes) { - limitExceeded = true; + throw e; } - return !limitExceeded; - } - - /** `true` when the writer was asked to write more than `maxBytes` bytes */ - public final boolean isLimitExceeded() { - return limitExceeded; - } - - @Override - public final void write(char[] cbuf, int off, int len) throws IOException { - if (canWriteBytes(len)) { - delegate.write(cbuf, off, len); - writtenBytes += len; - } - } - - @Override - public final void write(int c) throws IOException { - if (canWriteBytes(1)) { - delegate.write(c); - writtenBytes++; - } - } - - @Override - public final void write(String str, int off, int len) throws IOException { - if (canWriteBytes(len)) { - delegate.write(str, off, len); - writtenBytes += len; - } - } - - @Override - public final Writer append(CharSequence csq) throws IOException { - String str = (csq == null) ? "null" : csq.toString(); - write(str, 0, str.length()); - return this; - } - - @Override - public final Writer append(CharSequence csq, int start, int end) throws IOException { - String str = (csq == null) ? "null" : csq.subSequence(start, end).toString(); - write(str, 0, str.length()); - return this; - } - - @Override - public final Writer append(char c) throws IOException { - write(c); - return this; - } - - @Override - public final void flush() throws IOException { - delegate.flush(); - } - - @Override - public final void close() throws IOException { - delegate.close(); - } - - @Override - public final String toString() { - return delegate.toString(); } } } diff --git a/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataJson.java b/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataJson.java new file mode 100644 index 0000000000..888a5a1921 --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataJson.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.persistence; + +import java.util.Set; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; +import org.apache.polaris.service.catalog.iceberg.IcebergMetadataUtil; + +/** + * Represents a metadata.json file with its location and content + * + * @param location the location of the metadata.json itself + * @param content the content of the metadata.json + * @param tableLocations The locations that the table this metadata.json describes might be written. + * This includes the table base location as well as any data or metadata locations specified in + * the table properties + */ +public record MetadataJson(String location, String content, Set tableLocations) { + + /** Construct a record from a {@link TableMetadata} object */ + public static MetadataJson fromMetadata(TableMetadata metadata) { + Set tableLocations = IcebergMetadataUtil.getLocationsAllowedToBeAccessed(metadata); + return new MetadataJson( + metadata.metadataFileLocation(), TableMetadataParser.toJson(metadata), tableLocations); + } +} From d907378ef80c312f13a80d6bd6db0a6747d587cd Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 30 Apr 2025 15:23:06 -0700 Subject: [PATCH 66/88] test fix --- .../polaris/service/catalog/iceberg/IcebergCatalogHandler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index 37db12c6d9..74f9fdddcb 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -19,6 +19,7 @@ package org.apache.polaris.service.catalog.iceberg; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonRawValue; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import jakarta.annotation.Nonnull; @@ -1072,7 +1073,7 @@ public void close() throws Exception { public record StringLoadTableResponse( @JsonProperty("metadata-location") String metadataLocation, - @JsonProperty("metadata") String metadata, + @JsonProperty("metadata") @JsonRawValue String metadata, @JsonProperty("config") Map config, @JsonProperty("storage-credentials") List credentials) implements RESTResponse { From e434e21704ebf8024c0c7ae5b4a66d30ca543c12 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 30 Apr 2025 15:47:56 -0700 Subject: [PATCH 67/88] wip --- .../core/config/FeatureConfiguration.java | 2 +- .../catalog/iceberg/IcebergCatalog.java | 29 ++++--------------- .../iceberg/IcebergCatalogHandler.java | 5 ++++ 3 files changed, 12 insertions(+), 24 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java index 04452fe4f7..a463d0538a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java @@ -250,7 +250,7 @@ public static void enforceFeatureEnabledOrThrow( "If nonzero, the approximate max size a table's metadata can be in order to be cached in the persistence" + " layer. If zero, no metadata will be cached or served from the cache. If -1, all metadata" + " will be cached.") - .defaultValue(METADATA_CACHE_MAX_BYTES_NO_CACHING) + .defaultValue(METADATA_CACHE_MAX_BYTES_INFINITE_CACHING) .validation(value -> value >= -1) .buildFeatureConfiguration(); } diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index 385c23c6b0..43eb99acee 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -1399,24 +1399,7 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { .get(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY))) { // If location is changing then we must validate that the requested location is valid // for the storage configuration inherited under this entity's path. - Set dataLocations = new HashSet<>(); - dataLocations.add(metadata.location()); - if (metadata.properties().get(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY) - != null) { - dataLocations.add( - metadata - .properties() - .get(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY)); - } - if (metadata - .properties() - .get(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY) - != null) { - dataLocations.add( - metadata - .properties() - .get(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY)); - } + Set dataLocations = IcebergMetadataUtil.getLocationsAllowedToBeAccessed(metadata); validateLocationsForTableLike(tableIdentifier, dataLocations, resolvedStorageEntity); // also validate that the table location doesn't overlap an existing table dataLocations.forEach( @@ -1466,16 +1449,16 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { callContext.getPolarisCallContext(), catalogEntity, FeatureConfiguration.METADATA_CACHE_MAX_BYTES); - Optional metadataJsonOpt = + Optional metadataJsonToCache = switch (maxMetadataCacheBytes) { case FeatureConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING -> Optional.empty(); case FeatureConfiguration.METADATA_CACHE_MAX_BYTES_INFINITE_CACHING -> { yield Optional.of(TableMetadataParser.toJson(metadata)); } default -> { - metadataJsonOpt = Optional.of(TableMetadataParser.toJson(metadata)); - if (metadataJsonOpt.get().length() <= maxMetadataCacheBytes) { - yield Optional.of(TableMetadataParser.toJson(metadata)); + String rawMetadataJson = TableMetadataParser.toJson(metadata); + if (rawMetadataJson.length() < maxMetadataCacheBytes) { + yield Optional.of(rawMetadataJson); } else { yield Optional.empty(); } @@ -1498,7 +1481,7 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { .setBaseLocation(metadata.location()) .setMetadataLocation(newLocation); } - metadataJsonOpt.ifPresent(s -> builder.setMetadataContent(newLocation, s)); + metadataJsonToCache.ifPresent(s -> builder.setMetadataContent(newLocation, s)); entity = builder.build(); if (!Objects.equal(existingLocation, oldLocation)) { if (null == base) { diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index 74f9fdddcb..00bc62749c 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -1037,6 +1037,11 @@ public void renameView(RenameTableRequest request) { private @Nonnull RESTResponse filterResponseToSnapshots( RESTResponse restResponse, String snapshots) { + + // TODO push the filtering down into MetadataJson.fromMetadata + if (snapshots != null && !Set.of(SNAPSHOTS_ALL, SNAPSHOTS_REFS).contains(snapshots)) { + throw new IllegalArgumentException("Unrecognized snapshots: " + snapshots); + } if (restResponse instanceof LoadTableResponse loadTableResponse) { if (snapshots == null || snapshots.equalsIgnoreCase(SNAPSHOTS_ALL)) { return loadTableResponse; From 422ef42fab0e84c62a7910a0d3fb2bcf23574a48 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 30 Apr 2025 15:48:05 -0700 Subject: [PATCH 68/88] autolint --- .../apache/polaris/service/catalog/iceberg/IcebergCatalog.java | 1 - 1 file changed, 1 deletion(-) diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index 43eb99acee..864739465d 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -33,7 +33,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; From a829b078c9d540ca5b8736a3e6ee5bc6631594f2 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 30 Apr 2025 15:55:16 -0700 Subject: [PATCH 69/88] fixes --- .../catalog/iceberg/IcebergCatalog.java | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index 864739465d..b2652f40fe 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -1257,26 +1257,29 @@ public TableMetadata refresh() { return current(); } + /** With metadata caching, the `base` may not be exactly `current()` */ @Override public void commit(TableMetadata base, TableMetadata metadata) { - if (base == null) { - if (currentMetadata != null) { + // if the metadata is already out of date, reject it + if (currentMetadataLocation != null) { + if (base != null && !base.metadataFileLocation().equals(currentMetadataLocation)) { + throw new CommitFailedException("Cannot commit: stale table metadata"); + } else { // when current is non-null, the table exists. but when base is null, the commit is trying // to create the table throw new AlreadyExistsException("Table already exists: %s", fullTableName); } - } else if (base.metadataFileLocation() != null - && currentMetadata != null - && !base.metadataFileLocation().equals(currentMetadata.metadataFileLocation())) { - throw new CommitFailedException("Cannot commit: stale table metadata"); - } else if (base != currentMetadata) { - // This branch is different from BaseMetastoreTableOperations - LOGGER.debug( - "Base object differs from current metadata; proceeding because locations match"); - } else if (base.metadataFileLocation().equals(metadata.metadataFileLocation())) { - // if the metadata is not changed, return early - LOGGER.info("Nothing to commit."); - return; + } else if (base != metadata) { + if (base != null && metadata != null && + base.metadataFileLocation().equals(metadata.metadataFileLocation())) { + // if the metadata is not changed, return early + LOGGER.info("Nothing to commit."); + return; + } else { + // This branch is different from upstream due to metadata caching + LOGGER.debug( + "Base object differs from current metadata; proceeding because locations match"); + } } long start = System.currentTimeMillis(); From a4e17813043e3c215473cbee88b22e0b6e95fbba Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 30 Apr 2025 15:55:19 -0700 Subject: [PATCH 70/88] autolint --- .../polaris/service/catalog/iceberg/IcebergCatalog.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index b2652f40fe..e2c9b5f2f3 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -1270,8 +1270,9 @@ public void commit(TableMetadata base, TableMetadata metadata) { throw new AlreadyExistsException("Table already exists: %s", fullTableName); } } else if (base != metadata) { - if (base != null && metadata != null && - base.metadataFileLocation().equals(metadata.metadataFileLocation())) { + if (base != null + && metadata != null + && base.metadataFileLocation().equals(metadata.metadataFileLocation())) { // if the metadata is not changed, return early LOGGER.info("Nothing to commit."); return; From efddd271cbcbc08d22a45cb16baeb4453d368bfd Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 30 Apr 2025 16:51:27 -0700 Subject: [PATCH 71/88] snapshot filtering --- .../core/config/FeatureConfiguration.java | 9 +++ .../catalog/iceberg/IcebergCatalog.java | 56 +++++++++++++------ .../iceberg/IcebergCatalogAdapter.java | 4 +- .../iceberg/IcebergCatalogHandler.java | 56 +++++-------------- .../service/persistence/MetadataJson.java | 33 ++++++++++- 5 files changed, 96 insertions(+), 62 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java index a463d0538a..f6db210b19 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java @@ -253,4 +253,13 @@ public static void enforceFeatureEnabledOrThrow( .defaultValue(METADATA_CACHE_MAX_BYTES_INFINITE_CACHING) .validation(value -> value >= -1) .buildFeatureConfiguration(); + + public static final PolarisConfiguration ALWAYS_FILTER_SNAPSHOTS = + PolarisConfiguration.builder() + .key("ALWAYS_FILTER_SNAPSHOTS") + .description( + "If set, Polaris will always attempt to filter snapshots from a LoadTableResponse even when " + + "doing so requires additional serialization of the TableMetadata") + .defaultValue(true) + .buildFeatureConfiguration(); } diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index e2c9b5f2f3..a39868563c 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.catalog.iceberg; +import static org.apache.polaris.service.catalog.iceberg.IcebergCatalogHandler.SNAPSHOTS_ALL; import static org.apache.polaris.service.exception.IcebergExceptionMapper.isStorageProviderRetryableException; import com.google.common.annotations.VisibleForTesting; @@ -864,6 +865,21 @@ public Map getCredentialConfig( } public MetadataJson loadTableMetadataJson(TableIdentifier identifier) { + return loadTableMetadataJson(identifier, SNAPSHOTS_ALL); + } + + boolean shouldFilterSnapshots(String snapshots) { + if (snapshots == null || snapshots.equalsIgnoreCase(SNAPSHOTS_ALL)) { + return false; + } + return callContext + .getPolarisCallContext() + .getConfigurationStore() + .getConfiguration( + callContext.getPolarisCallContext(), FeatureConfiguration.ALWAYS_FILTER_SNAPSHOTS); + } + + public MetadataJson loadTableMetadataJson(TableIdentifier identifier, String snapshots) { int maxMetadataCacheBytes = callContext .getPolarisCallContext() @@ -875,7 +891,7 @@ public MetadataJson loadTableMetadataJson(TableIdentifier identifier) { return loadTableMetadata(loadTable(identifier)); }; if (maxMetadataCacheBytes == FeatureConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING) { - return MetadataJson.fromMetadata(fallback.get()); + return MetadataJson.fromMetadata(fallback.get(), snapshots); } else { return MetadataCacheManager.loadTableMetadataJson( identifier, @@ -1261,25 +1277,26 @@ public TableMetadata refresh() { @Override public void commit(TableMetadata base, TableMetadata metadata) { // if the metadata is already out of date, reject it - if (currentMetadataLocation != null) { - if (base != null && !base.metadataFileLocation().equals(currentMetadataLocation)) { - throw new CommitFailedException("Cannot commit: stale table metadata"); - } else { + if (base == null) { + if (current() != null) { // when current is non-null, the table exists. but when base is null, the commit is trying // to create the table throw new AlreadyExistsException("Table already exists: %s", fullTableName); } - } else if (base != metadata) { + } else if (current() != null && !current().metadataFileLocation().equals(base.metadataFileLocation())) { + throw new CommitFailedException("Cannot commit: stale table metadata"); + } + // if the metadata is not changed, return early + if (base == metadata) { + LOGGER.info("Nothing to commit."); + return; + } else { if (base != null && metadata != null && base.metadataFileLocation().equals(metadata.metadataFileLocation())) { // if the metadata is not changed, return early LOGGER.info("Nothing to commit."); return; - } else { - // This branch is different from upstream due to metadata caching - LOGGER.debug( - "Base object differs from current metadata; proceeding because locations match"); } } @@ -1503,12 +1520,7 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { // We diverge from `BaseMetastoreTableOperations` in the below code block if (makeMetadataCurrentOnCommit) { - currentMetadata = - TableMetadata.buildFrom(metadata) - .withMetadataLocation(newLocation) - .discardChanges() - .build(); - currentMetadataLocation = newLocation; + setCurrentMetadata(newLocation, metadata); } if (null == existingLocation) { @@ -1518,6 +1530,18 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { } } + public void setCurrentMetadata(String metadataLocation, TableMetadata metadata) { + if (metadata == null) { + return; + } + currentMetadata = + TableMetadata.buildFrom(metadata) + .withMetadataLocation(metadataLocation) + .discardChanges() + .build(); + currentMetadataLocation = metadataLocation; + } + @Override public TableOperations temp(TableMetadata uncommittedMetadata) { return new TableOperations() { diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java index d27fdc5a9c..d6b5c0f83e 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java @@ -346,8 +346,8 @@ public Response createTable( Response.ok(response), response, namespace, createTableRequest.name()) .build(); } else { - RESTResponse response = - catalog.createTableDirectWithWriteDelegation(ns, createTableRequest); + RESTResponse response = catalog + .createTableDirectWithWriteDelegation(ns, createTableRequest); return tryInsertETagHeader( Response.ok(response), response, namespace, createTableRequest.name()) .build(); diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index 00bc62749c..b8ed7b17a4 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -590,14 +590,16 @@ public Optional loadTableIfStale( } } - final RESTResponse rawResponse; if (baseCatalog instanceof IcebergCatalog icebergCatalog) { MetadataJson metadataJson = icebergCatalog.loadTableMetadataJson(tableIdentifier); - rawResponse = new StringLoadTableResponse(metadataJson); + if (icebergCatalog.shouldFilterSnapshots(snapshots)) { + TableMetadata metadata = TableMetadataParser.fromJson(metadataJson.content()); + metadataJson = MetadataJson.fromMetadata(metadata, snapshots); // round-trip to filter + } + return Optional.of(new StringLoadTableResponse(metadataJson)); } else { - rawResponse = CatalogHandlers.loadTable(baseCatalog, tableIdentifier); + return Optional.of(CatalogHandlers.loadTable(baseCatalog, tableIdentifier)); } - return Optional.of(filterResponseToSnapshots(rawResponse, snapshots)); } public RESTResponse loadTableWithAccessDelegation( @@ -687,18 +689,22 @@ public Optional loadTableWithAccessDelegationIfStale( } } } - MetadataJson tableMetadata = null; + MetadataJson metadataJson = null; if (baseCatalog instanceof IcebergCatalog icebergCatalog) { - tableMetadata = icebergCatalog.loadTableMetadataJson(tableIdentifier); + metadataJson = icebergCatalog.loadTableMetadataJson(tableIdentifier, snapshots); + if (icebergCatalog.shouldFilterSnapshots(snapshots)) { + TableMetadata metadata = TableMetadataParser.fromJson(metadataJson.content()); + metadataJson = MetadataJson.fromMetadata(metadata, snapshots); // round-trip to filter + } } // The metadata failed to load - if (tableMetadata == null) { + if (metadataJson == null) { throw new NoSuchTableException("Table does not exist: %s", tableIdentifier.toString()); } else { return Optional.of( buildLoadTableResponseWithDelegationCredentials( - tableIdentifier, tableMetadata, actionsRequested)); + tableIdentifier, metadataJson, actionsRequested)); } } @@ -1035,40 +1041,6 @@ public void renameView(RenameTableRequest request) { CatalogHandlers.renameView(viewCatalog, request); } - private @Nonnull RESTResponse filterResponseToSnapshots( - RESTResponse restResponse, String snapshots) { - - // TODO push the filtering down into MetadataJson.fromMetadata - if (snapshots != null && !Set.of(SNAPSHOTS_ALL, SNAPSHOTS_REFS).contains(snapshots)) { - throw new IllegalArgumentException("Unrecognized snapshots: " + snapshots); - } - if (restResponse instanceof LoadTableResponse loadTableResponse) { - if (snapshots == null || snapshots.equalsIgnoreCase(SNAPSHOTS_ALL)) { - return loadTableResponse; - } else if (snapshots.equalsIgnoreCase(SNAPSHOTS_REFS)) { - TableMetadata metadata = loadTableResponse.tableMetadata(); - - Set referencedSnapshotIds = - metadata.refs().values().stream() - .map(SnapshotRef::snapshotId) - .collect(Collectors.toSet()); - - TableMetadata filteredMetadata = - metadata.removeSnapshotsIf(s -> !referencedSnapshotIds.contains(s.snapshotId())); - - return LoadTableResponse.builder() - .withTableMetadata(filteredMetadata) - .addAllConfig(loadTableResponse.config()) - .addAllCredentials(loadTableResponse.credentials()) - .build(); - } else { - throw new IllegalArgumentException("Unrecognized snapshots: " + snapshots); - } - } else { - return restResponse; - } - } - @Override public void close() throws Exception { if (baseCatalog instanceof Closeable closeable) { diff --git a/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataJson.java b/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataJson.java index 888a5a1921..a32a9960bc 100644 --- a/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataJson.java +++ b/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataJson.java @@ -18,11 +18,19 @@ */ package org.apache.polaris.service.persistence; +import java.util.HashSet; import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.iceberg.SnapshotRef; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.rest.responses.LoadTableResponse; import org.apache.polaris.service.catalog.iceberg.IcebergMetadataUtil; +import static org.apache.polaris.service.catalog.iceberg.IcebergCatalogHandler.SNAPSHOTS_ALL; +import static org.apache.polaris.service.catalog.iceberg.IcebergCatalogHandler.SNAPSHOTS_REFS; + /** * Represents a metadata.json file with its location and content * @@ -36,8 +44,29 @@ public record MetadataJson(String location, String content, Set tableLoc /** Construct a record from a {@link TableMetadata} object */ public static MetadataJson fromMetadata(TableMetadata metadata) { - Set tableLocations = IcebergMetadataUtil.getLocationsAllowedToBeAccessed(metadata); + return fromMetadata(metadata, SNAPSHOTS_ALL); + } + + /** See {@link MetadataJson#fromMetadata(TableMetadata)} */ + public static MetadataJson fromMetadata(TableMetadata metadata, String snapshots) { + final TableMetadata filteredMetadata; + if (snapshots != null && !snapshots.equalsIgnoreCase(SNAPSHOTS_ALL)) { + if (snapshots.equalsIgnoreCase(SNAPSHOTS_REFS)) { + Set referencedSnapshotIds = + metadata.refs().values().stream() + .map(SnapshotRef::snapshotId) + .collect(Collectors.toSet()); + + filteredMetadata = + metadata.removeSnapshotsIf(s -> !referencedSnapshotIds.contains(s.snapshotId())); + } else { + throw new IllegalArgumentException("Unrecognized snapshots: " + snapshots); + } + } else { + filteredMetadata = metadata; + } + Set tableLocations = IcebergMetadataUtil.getLocationsAllowedToBeAccessed(filteredMetadata); return new MetadataJson( - metadata.metadataFileLocation(), TableMetadataParser.toJson(metadata), tableLocations); + metadata.metadataFileLocation(), TableMetadataParser.toJson(filteredMetadata), tableLocations); } } From d78fdaf5302cc62581f8baa8fa9844135e3859b3 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 30 Apr 2025 16:51:31 -0700 Subject: [PATCH 72/88] autolint --- .../core/config/FeatureConfiguration.java | 4 ++-- .../service/catalog/iceberg/IcebergCatalog.java | 3 ++- .../catalog/iceberg/IcebergCatalogAdapter.java | 4 ++-- .../catalog/iceberg/IcebergCatalogHandler.java | 2 -- .../service/persistence/MetadataJson.java | 16 ++++++++-------- 5 files changed, 14 insertions(+), 15 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java index f6db210b19..131ab9dbbf 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java @@ -258,8 +258,8 @@ public static void enforceFeatureEnabledOrThrow( PolarisConfiguration.builder() .key("ALWAYS_FILTER_SNAPSHOTS") .description( - "If set, Polaris will always attempt to filter snapshots from a LoadTableResponse even when " + - "doing so requires additional serialization of the TableMetadata") + "If set, Polaris will always attempt to filter snapshots from a LoadTableResponse even when " + + "doing so requires additional serialization of the TableMetadata") .defaultValue(true) .buildFeatureConfiguration(); } diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index a39868563c..a444205449 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -1283,7 +1283,8 @@ public void commit(TableMetadata base, TableMetadata metadata) { // to create the table throw new AlreadyExistsException("Table already exists: %s", fullTableName); } - } else if (current() != null && !current().metadataFileLocation().equals(base.metadataFileLocation())) { + } else if (current() != null + && !current().metadataFileLocation().equals(base.metadataFileLocation())) { throw new CommitFailedException("Cannot commit: stale table metadata"); } // if the metadata is not changed, return early diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java index d6b5c0f83e..d27fdc5a9c 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java @@ -346,8 +346,8 @@ public Response createTable( Response.ok(response), response, namespace, createTableRequest.name()) .build(); } else { - RESTResponse response = catalog - .createTableDirectWithWriteDelegation(ns, createTableRequest); + RESTResponse response = + catalog.createTableDirectWithWriteDelegation(ns, createTableRequest); return tryInsertETagHeader( Response.ok(response), response, namespace, createTableRequest.name()) .build(); diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index b8ed7b17a4..dcb69c91c1 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.annotation.JsonRawValue; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; -import jakarta.annotation.Nonnull; import jakarta.ws.rs.core.SecurityContext; import java.io.Closeable; import java.time.OffsetDateTime; @@ -39,7 +38,6 @@ import org.apache.iceberg.BaseTable; import org.apache.iceberg.MetadataUpdate; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.SnapshotRef; import org.apache.iceberg.SortOrder; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; diff --git a/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataJson.java b/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataJson.java index a32a9960bc..ae0ce858f7 100644 --- a/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataJson.java +++ b/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataJson.java @@ -18,19 +18,16 @@ */ package org.apache.polaris.service.persistence; -import java.util.HashSet; +import static org.apache.polaris.service.catalog.iceberg.IcebergCatalogHandler.SNAPSHOTS_ALL; +import static org.apache.polaris.service.catalog.iceberg.IcebergCatalogHandler.SNAPSHOTS_REFS; + import java.util.Set; import java.util.stream.Collectors; - import org.apache.iceberg.SnapshotRef; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableMetadataParser; -import org.apache.iceberg.rest.responses.LoadTableResponse; import org.apache.polaris.service.catalog.iceberg.IcebergMetadataUtil; -import static org.apache.polaris.service.catalog.iceberg.IcebergCatalogHandler.SNAPSHOTS_ALL; -import static org.apache.polaris.service.catalog.iceberg.IcebergCatalogHandler.SNAPSHOTS_REFS; - /** * Represents a metadata.json file with its location and content * @@ -65,8 +62,11 @@ public static MetadataJson fromMetadata(TableMetadata metadata, String snapshots } else { filteredMetadata = metadata; } - Set tableLocations = IcebergMetadataUtil.getLocationsAllowedToBeAccessed(filteredMetadata); + Set tableLocations = + IcebergMetadataUtil.getLocationsAllowedToBeAccessed(filteredMetadata); return new MetadataJson( - metadata.metadataFileLocation(), TableMetadataParser.toJson(filteredMetadata), tableLocations); + metadata.metadataFileLocation(), + TableMetadataParser.toJson(filteredMetadata), + tableLocations); } } From efb3be211cd4597eac398ec27004bf51c123a16f Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 30 Apr 2025 17:22:39 -0700 Subject: [PATCH 73/88] fix --- .../service/catalog/iceberg/IcebergCatalog.java | 1 + .../catalog/iceberg/IcebergCatalogHandler.java | 12 +++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index a444205449..7302b59ab4 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -1285,6 +1285,7 @@ public void commit(TableMetadata base, TableMetadata metadata) { } } else if (current() != null && !current().metadataFileLocation().equals(base.metadataFileLocation())) { + System.out.println("#### " + current().metadataFileLocation() + " vs " + base.metadataFileLocation()); throw new CommitFailedException("Cannot commit: stale table metadata"); } // if the metadata is not changed, return early diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index dcb69c91c1..309a797297 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -884,18 +884,20 @@ public void commitTransaction(CommitTransactionRequest commitTransactionRequest) final TableMetadata currentMetadata; final TableOperations tableOps; if (baseCatalog instanceof IcebergCatalog icebergCatalog) { - currentMetadata = - TableMetadataParser.fromJson( - icebergCatalog.loadTableMetadataJson(change.identifier()).content()); tableOps = icebergCatalog.newTableOps(change.identifier()); + MetadataJson metadataJson = icebergCatalog.loadTableMetadataJson(change.identifier()); + currentMetadata = TableMetadataParser.fromJson(metadataJson.content()); + // Update tableOps.current() to reflect the cached metadata + if (tableOps instanceof IcebergCatalog.BasePolarisTableOperations bpto) { + bpto.setCurrentMetadata(metadataJson.location(), currentMetadata); + bpto.shouldRefresh = false; + } } else { final Table table = baseCatalog.loadTable(change.identifier()); - if (!(table instanceof BaseTable)) { throw new IllegalStateException( "Cannot wrap catalog that does not produce BaseTable"); } - tableOps = ((BaseTable) table).operations(); currentMetadata = tableOps.current(); } From b98e8454a5bc5114e9315be1a49a03850db85008 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 30 Apr 2025 17:31:42 -0700 Subject: [PATCH 74/88] stable --- .../apache/polaris/core/config/FeatureConfiguration.java | 2 +- .../polaris/service/catalog/iceberg/IcebergCatalog.java | 6 ++++-- .../service/catalog/iceberg/IcebergCatalogHandler.java | 9 +++++++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java index 131ab9dbbf..66e5209601 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java @@ -250,7 +250,7 @@ public static void enforceFeatureEnabledOrThrow( "If nonzero, the approximate max size a table's metadata can be in order to be cached in the persistence" + " layer. If zero, no metadata will be cached or served from the cache. If -1, all metadata" + " will be cached.") - .defaultValue(METADATA_CACHE_MAX_BYTES_INFINITE_CACHING) + .defaultValue(METADATA_CACHE_MAX_BYTES_NO_CACHING) .validation(value -> value >= -1) .buildFeatureConfiguration(); diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index 7302b59ab4..386ac5682d 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -1273,7 +1273,10 @@ public TableMetadata refresh() { return current(); } - /** With metadata caching, the `base` may not be exactly `current()` */ + /** + * With metadata caching, the `base` may not be exactly `current()` by reference so we compare + * locations instead + */ @Override public void commit(TableMetadata base, TableMetadata metadata) { // if the metadata is already out of date, reject it @@ -1285,7 +1288,6 @@ public void commit(TableMetadata base, TableMetadata metadata) { } } else if (current() != null && !current().metadataFileLocation().equals(base.metadataFileLocation())) { - System.out.println("#### " + current().metadataFileLocation() + " vs " + base.metadataFileLocation()); throw new CommitFailedException("Cannot commit: stale table metadata"); } // if the metadata is not changed, return early diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index 309a797297..f907df47b3 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -881,12 +881,17 @@ public void commitTransaction(CommitTransactionRequest commitTransactionRequest) commitTransactionRequest.tableChanges().stream() .forEach( change -> { + // TODO we are still loading metadata redundantly against the same table final TableMetadata currentMetadata; final TableOperations tableOps; if (baseCatalog instanceof IcebergCatalog icebergCatalog) { tableOps = icebergCatalog.newTableOps(change.identifier()); - MetadataJson metadataJson = icebergCatalog.loadTableMetadataJson(change.identifier()); - currentMetadata = TableMetadataParser.fromJson(metadataJson.content()); + MetadataJson metadataJson = + icebergCatalog.loadTableMetadataJson(change.identifier()); + currentMetadata = + TableMetadata.buildFrom(TableMetadataParser.fromJson(metadataJson.content())) + .withMetadataLocation(metadataJson.location()) + .build(); // Update tableOps.current() to reflect the cached metadata if (tableOps instanceof IcebergCatalog.BasePolarisTableOperations bpto) { bpto.setCurrentMetadata(metadataJson.location(), currentMetadata); From fea135d9f7b0d2e1dcb532f4872703fed4a96213 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Tue, 13 May 2025 19:20:07 -0700 Subject: [PATCH 75/88] improve null check --- .../polaris/core/config/PolarisConfigurationStore.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/PolarisConfigurationStore.java b/polaris-core/src/main/java/org/apache/polaris/core/config/PolarisConfigurationStore.java index acb171ce30..662a88d72d 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/PolarisConfigurationStore.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/PolarisConfigurationStore.java @@ -118,9 +118,14 @@ public interface PolarisConfigurationStore { PolarisConfiguration config) { if (config.hasCatalogConfig() || config.hasCatalogConfigUnsafe()) { Map propertiesMap = catalogEntity.getPropertiesAsMap(); - String propertyValue = propertiesMap.get(config.catalogConfig()); + String propertyValue = null; + if (config.hasCatalogConfig()) { + propertyValue = propertiesMap.get(config.catalogConfig()); + } if (propertyValue == null) { - propertyValue = propertiesMap.get(config.catalogConfigUnsafe()); + if (config.hasCatalogConfigUnsafe()) { + propertyValue = propertiesMap.get(config.catalogConfigUnsafe()); + } if (propertyValue != null) { LOGGER.warn( String.format( From 99e2a3768eb815a10cc4d3a9081d5c27f7e349ed Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Tue, 13 May 2025 19:31:02 -0700 Subject: [PATCH 76/88] add a test --- .../config/DefaultConfigurationStoreTest.java | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/config/DefaultConfigurationStoreTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/config/DefaultConfigurationStoreTest.java index e533d52142..cd72a53a24 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/config/DefaultConfigurationStoreTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/config/DefaultConfigurationStoreTest.java @@ -30,13 +30,16 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.service.config.DefaultConfigurationStore; import org.apache.polaris.service.config.FeaturesConfiguration; import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; +import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; @@ -230,4 +233,51 @@ public void testInjectedFeaturesConfiguration() { assertThat(featuresConfiguration.realmOverrides().get(realmOne).overrides()) .containsKey(falseByDefaultKey); } + + @Test + public void testRegisterAndUseFeatureConfigurations() { + String prefix = "testRegisterAndUseFeatureConfigurations"; + + FeatureConfiguration safeConfig = FeatureConfiguration.builder() + .key(String.format("%s_safe", prefix)) + .catalogConfig(String.format("polaris.config.%s.safe", prefix)) + .defaultValue(true) + .description(prefix) + .buildFeatureConfiguration(); + + FeatureConfiguration unsafeConfig =FeatureConfiguration.builder() + .key(String.format("%s_unsafe", prefix)) + .catalogConfigUnsafe(String.format("%s.unsafe", prefix)) + .defaultValue(true) + .description(prefix) + .buildFeatureConfiguration(); + + FeatureConfiguration bothConfig = FeatureConfiguration.builder() + .key(String.format("%s_both", prefix)) + .catalogConfig(String.format("polaris.config.%s.both", prefix)) + .catalogConfigUnsafe(String.format("%s.both", prefix)) + .defaultValue(true) + .description(prefix) + .buildFeatureConfiguration(); + + CatalogEntity catalog = new CatalogEntity.Builder().build(); + + Assertions.assertThat(configurationStore + .getConfiguration( + polarisContext, + catalog, + safeConfig)).isTrue(); + + Assertions.assertThat(configurationStore + .getConfiguration( + polarisContext, + catalog, + unsafeConfig)).isTrue(); + + Assertions.assertThat(configurationStore + .getConfiguration( + polarisContext, + catalog, + bothConfig)).isTrue(); + } } From 29082076cd38a988584b6195d1547b99f7f807ed Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Tue, 13 May 2025 19:31:05 -0700 Subject: [PATCH 77/88] autolint --- .../config/DefaultConfigurationStoreTest.java | 71 +++++++++---------- 1 file changed, 33 insertions(+), 38 deletions(-) diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/config/DefaultConfigurationStoreTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/config/DefaultConfigurationStoreTest.java index cd72a53a24..ddd1026d97 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/config/DefaultConfigurationStoreTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/config/DefaultConfigurationStoreTest.java @@ -238,46 +238,41 @@ public void testInjectedFeaturesConfiguration() { public void testRegisterAndUseFeatureConfigurations() { String prefix = "testRegisterAndUseFeatureConfigurations"; - FeatureConfiguration safeConfig = FeatureConfiguration.builder() - .key(String.format("%s_safe", prefix)) - .catalogConfig(String.format("polaris.config.%s.safe", prefix)) - .defaultValue(true) - .description(prefix) - .buildFeatureConfiguration(); - - FeatureConfiguration unsafeConfig =FeatureConfiguration.builder() - .key(String.format("%s_unsafe", prefix)) - .catalogConfigUnsafe(String.format("%s.unsafe", prefix)) - .defaultValue(true) - .description(prefix) - .buildFeatureConfiguration(); - - FeatureConfiguration bothConfig = FeatureConfiguration.builder() - .key(String.format("%s_both", prefix)) - .catalogConfig(String.format("polaris.config.%s.both", prefix)) - .catalogConfigUnsafe(String.format("%s.both", prefix)) - .defaultValue(true) - .description(prefix) - .buildFeatureConfiguration(); + FeatureConfiguration safeConfig = + FeatureConfiguration.builder() + .key(String.format("%s_safe", prefix)) + .catalogConfig(String.format("polaris.config.%s.safe", prefix)) + .defaultValue(true) + .description(prefix) + .buildFeatureConfiguration(); + + FeatureConfiguration unsafeConfig = + FeatureConfiguration.builder() + .key(String.format("%s_unsafe", prefix)) + .catalogConfigUnsafe(String.format("%s.unsafe", prefix)) + .defaultValue(true) + .description(prefix) + .buildFeatureConfiguration(); + + FeatureConfiguration bothConfig = + FeatureConfiguration.builder() + .key(String.format("%s_both", prefix)) + .catalogConfig(String.format("polaris.config.%s.both", prefix)) + .catalogConfigUnsafe(String.format("%s.both", prefix)) + .defaultValue(true) + .description(prefix) + .buildFeatureConfiguration(); CatalogEntity catalog = new CatalogEntity.Builder().build(); - Assertions.assertThat(configurationStore - .getConfiguration( - polarisContext, - catalog, - safeConfig)).isTrue(); - - Assertions.assertThat(configurationStore - .getConfiguration( - polarisContext, - catalog, - unsafeConfig)).isTrue(); - - Assertions.assertThat(configurationStore - .getConfiguration( - polarisContext, - catalog, - bothConfig)).isTrue(); + Assertions.assertThat(configurationStore.getConfiguration(polarisContext, catalog, safeConfig)) + .isTrue(); + + Assertions.assertThat( + configurationStore.getConfiguration(polarisContext, catalog, unsafeConfig)) + .isTrue(); + + Assertions.assertThat(configurationStore.getConfiguration(polarisContext, catalog, bothConfig)) + .isTrue(); } } From cbf1b86efdf1e28701f8bc7845095ff68c40d234 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 15 May 2025 21:28:07 -0700 Subject: [PATCH 78/88] changes per review --- .../polaris/service/catalog/iceberg/IcebergCatalog.java | 5 +---- .../service/catalog/iceberg/IcebergCatalogAdapter.java | 8 ++++---- .../polaris/service/persistence/MetadataCacheManager.java | 4 ++-- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index 386ac5682d..5f7864ead0 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -1481,7 +1481,7 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { } default -> { String rawMetadataJson = TableMetadataParser.toJson(metadata); - if (rawMetadataJson.length() < maxMetadataCacheBytes) { + if (rawMetadataJson.length() * 2 < maxMetadataCacheBytes) { yield Optional.of(rawMetadataJson); } else { yield Optional.empty(); @@ -1535,9 +1535,6 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { } public void setCurrentMetadata(String metadataLocation, TableMetadata metadata) { - if (metadata == null) { - return; - } currentMetadata = TableMetadata.buildFrom(metadata) .withMetadataLocation(metadataLocation) diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java index d27fdc5a9c..45060383a1 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java @@ -244,8 +244,8 @@ private Response.ResponseBuilder tryInsertETagHeader( final String metadataLocation; if (response instanceof LoadTableResponse loadTableResponse) { metadataLocation = loadTableResponse.metadataLocation(); - } else if (response instanceof IcebergCatalogHandler.StringLoadTableResponse sltr) { - metadataLocation = sltr.metadataLocation(); + } else if (response instanceof IcebergCatalogHandler.StringLoadTableResponse stringResponse) { + metadataLocation = stringResponse.metadataLocation(); } else { throw new IllegalStateException("Cannot build etag from: " + response); } @@ -554,8 +554,8 @@ public Response loadCredentials( final List credentials; if (restResponse instanceof LoadTableResponse loadTableResponse) { credentials = loadTableResponse.credentials(); - } else if (restResponse instanceof IcebergCatalogHandler.StringLoadTableResponse sltr) { - credentials = sltr.credentials(); + } else if (restResponse instanceof IcebergCatalogHandler.StringLoadTableResponse stringResponse) { + credentials = stringResponse.credentials(); } else { throw new IllegalStateException("Cannot extract credentials from " + restResponse); } diff --git a/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 6811d4686c..d48b43a043 100644 --- a/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -58,7 +58,6 @@ public static MetadataJson loadTableMetadataJson( PolarisMetaStoreManager metastoreManager, PolarisResolutionManifestCatalogView resolvedEntityView, Supplier fallback) { - LOGGER.debug(String.format("Loading cached metadata for %s", tableIdentifier)); PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getResolvedPath( tableIdentifier, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ICEBERG_TABLE); @@ -66,6 +65,7 @@ public static MetadataJson loadTableMetadataJson( if (resolvedEntities == null) { return MetadataJson.fromMetadata(fallback.get()); } + LOGGER.debug(String.format("Loading cached metadata for %s", tableIdentifier)); IcebergTableLikeEntity tableLikeEntity = IcebergTableLikeEntity.of(resolvedEntities.getRawLeafEntity()); String cacheContent = tableLikeEntity.getMetadataCacheContent(); @@ -141,7 +141,7 @@ private static EntityResult cacheTableMetadataJson( // PersistenceException (& other extension-specific exceptions) may not be in scope, // but we can make a best-effort attempt to swallow it and just forego caching if (e.toString().contains("PersistenceException")) { - LOGGER.debug( + LOGGER.warn( String.format( "Encountered an error while caching %s: %s", tableLikeEntity.getTableIdentifier(), e)); From d8ba2726281122068d65e3d6d83cf03d48e83061 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 15 May 2025 21:28:10 -0700 Subject: [PATCH 79/88] autolint --- .../polaris/service/catalog/iceberg/IcebergCatalogAdapter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java index 45060383a1..27df67ccdc 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java @@ -554,7 +554,8 @@ public Response loadCredentials( final List credentials; if (restResponse instanceof LoadTableResponse loadTableResponse) { credentials = loadTableResponse.credentials(); - } else if (restResponse instanceof IcebergCatalogHandler.StringLoadTableResponse stringResponse) { + } else if (restResponse + instanceof IcebergCatalogHandler.StringLoadTableResponse stringResponse) { credentials = stringResponse.credentials(); } else { throw new IllegalStateException("Cannot extract credentials from " + restResponse); From 8e98c17f3956c953ab2a1e1d45c83b4aaa4b32a6 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 16 May 2025 13:15:54 -0700 Subject: [PATCH 80/88] re-stabilize --- .../entity/table/IcebergTableLikeEntity.java | 2 +- .../IcebergCatalogHandlerAuthzTest.java | 5 +- .../quarkus/catalog/IcebergCatalogTest.java | 26 ++-- .../catalog/iceberg/IcebergCatalog.java | 23 +--- .../iceberg/IcebergCatalogAdapter.java | 5 - .../iceberg/IcebergCatalogHandler.java | 127 +++++++++--------- .../persistence/MetadataCacheManager.java | 49 +++---- .../service/persistence/MetadataJson.java | 72 ---------- 8 files changed, 99 insertions(+), 210 deletions(-) delete mode 100644 service/common/src/main/java/org/apache/polaris/service/persistence/MetadataJson.java diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/table/IcebergTableLikeEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/table/IcebergTableLikeEntity.java index b998c19a39..6d48a55051 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/table/IcebergTableLikeEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/table/IcebergTableLikeEntity.java @@ -40,7 +40,7 @@ public class IcebergTableLikeEntity extends TableLikeEntity { // For applicable types, this key on the "internalProperties" map will return the content of the // metadata.json file located at `METADATA_CACHE_LOCATION_KEY` - private static final String METADATA_CACHE_CONTENT_KEY = "metadata-cache-content"; + public static final String METADATA_CACHE_CONTENT_KEY = "metadata-cache-content"; // For applicable types, this key on the "internalProperties" map will return the location of the // `metadata.json` that is cached in `METADATA_CACHE_CONTENT_KEY`. This will often match the diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java index 8e0ab67665..400e8e3b7a 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java @@ -772,10 +772,7 @@ public void testCreateTableStagedWithWriteDelegationInsufficientPermissions() { } private static String getMetadataLocation(RESTResponse resp) { - final String metadataLocation = - resp instanceof IcebergCatalogHandler.StringLoadTableResponse - ? ((IcebergCatalogHandler.StringLoadTableResponse) resp).metadataLocation() - : ((LoadTableResponse) resp).metadataLocation(); + final String metadataLocation = ((LoadTableResponse) resp).metadataLocation(); return metadataLocation; } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java index 0ec2ac43dd..0279f30235 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java @@ -129,7 +129,6 @@ import org.apache.polaris.service.exception.FakeAzureHttpResponse; import org.apache.polaris.service.exception.IcebergExceptionMapper; import org.apache.polaris.service.persistence.MetadataCacheManager; -import org.apache.polaris.service.persistence.MetadataJson; import org.apache.polaris.service.quarkus.config.QuarkusReservedProperties; import org.apache.polaris.service.quarkus.test.TestData; import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; @@ -182,7 +181,9 @@ public Map getConfigOverrides() { "polaris.features.\"LIST_PAGINATION_ENABLED\"", "true", "polaris.event-listener.type", - "test"); + "test", + "polaris.features." + FeatureConfiguration.METADATA_CACHE_MAX_BYTES.key, + "10000"); } } @@ -1820,11 +1821,11 @@ public void testMetadataCachingWithManualFallback() { Schema schema = buildSchema(10); catalog.createNamespace(namespace); - Table createdTable = catalog.createTable(tableIdentifier, schema); - TableMetadata originalMetadata = ((BaseTable) createdTable).operations().current(); + catalog.createTable(tableIdentifier, schema); + TableMetadata originalMetadata = catalog.loadTableMetadata(tableIdentifier); - MetadataJson cachedMetadataJson = - MetadataCacheManager.loadTableMetadataJson( + TableMetadata cachedMetadata = + MetadataCacheManager.loadTableMetadata( tableIdentifier, Integer.MAX_VALUE, polarisContext, @@ -1835,7 +1836,7 @@ public void testMetadataCachingWithManualFallback() { }); // The content should match what was cached - Assertions.assertThat(cachedMetadataJson.content()) + Assertions.assertThat(TableMetadataParser.toJson(cachedMetadata)) .isEqualTo(TableMetadataParser.toJson(originalMetadata)); // Update the table @@ -1844,8 +1845,8 @@ public void testMetadataCachingWithManualFallback() { tableOps.commit(tableOps.current(), updatedMetadata); // Read from the cache; it should detect a change due to the update and load the new fallback - MetadataJson reloadedMetadataJson = - MetadataCacheManager.loadTableMetadataJson( + TableMetadata reloadedMetadata = + MetadataCacheManager.loadTableMetadata( tableIdentifier, Integer.MAX_VALUE, polarisContext, @@ -1856,10 +1857,9 @@ public void testMetadataCachingWithManualFallback() { "Fell back even though a cache entry should be updated on write"); }); - Assertions.assertThat(reloadedMetadataJson).isNotSameAs(cachedMetadataJson); - Assertions.assertThat( - TableMetadataParser.fromJson(reloadedMetadataJson.content()).schema().columns().size()) - .isEqualTo(100); + Assertions.assertThat(TableMetadataParser.toJson(reloadedMetadata)) + .isNotSameAs(TableMetadataParser.toJson(cachedMetadata)); + Assertions.assertThat(reloadedMetadata.schema().columns().size()).isEqualTo(100); } @Test diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index da5db485a4..e18c472d06 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -18,7 +18,6 @@ */ package org.apache.polaris.service.catalog.iceberg; -import static org.apache.polaris.service.catalog.iceberg.IcebergCatalogHandler.SNAPSHOTS_ALL; import static org.apache.polaris.service.exception.IcebergExceptionMapper.isStorageProviderRetryableException; import com.google.common.annotations.VisibleForTesting; @@ -136,7 +135,6 @@ import org.apache.polaris.service.events.BeforeViewRefreshedEvent; import org.apache.polaris.service.events.PolarisEventListener; import org.apache.polaris.service.persistence.MetadataCacheManager; -import org.apache.polaris.service.persistence.MetadataJson; import org.apache.polaris.service.task.TaskExecutor; import org.apache.polaris.service.types.NotificationRequest; import org.apache.polaris.service.types.NotificationType; @@ -910,22 +908,7 @@ public AccessConfig getAccessConfig( storageInfo.get()); } - public MetadataJson loadTableMetadataJson(TableIdentifier identifier) { - return loadTableMetadataJson(identifier, SNAPSHOTS_ALL); - } - - boolean shouldFilterSnapshots(String snapshots) { - if (snapshots == null || snapshots.equalsIgnoreCase(SNAPSHOTS_ALL)) { - return false; - } - return callContext - .getPolarisCallContext() - .getConfigurationStore() - .getConfiguration( - callContext.getPolarisCallContext(), FeatureConfiguration.ALWAYS_FILTER_SNAPSHOTS); - } - - public MetadataJson loadTableMetadataJson(TableIdentifier identifier, String snapshots) { + public TableMetadata loadTableMetadata(TableIdentifier identifier) { int maxMetadataCacheBytes = callContext .getPolarisCallContext() @@ -937,9 +920,9 @@ public MetadataJson loadTableMetadataJson(TableIdentifier identifier, String sna return loadTableMetadata(loadTable(identifier)); }; if (maxMetadataCacheBytes == FeatureConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING) { - return MetadataJson.fromMetadata(fallback.get(), snapshots); + return fallback.get(); } else { - return MetadataCacheManager.loadTableMetadataJson( + return MetadataCacheManager.loadTableMetadata( identifier, maxMetadataCacheBytes, callContext.getPolarisCallContext(), diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java index 5f8a0d60da..c73a427abd 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java @@ -256,8 +256,6 @@ private Response.ResponseBuilder tryInsertETagHeader( final String metadataLocation; if (response instanceof LoadTableResponse loadTableResponse) { metadataLocation = loadTableResponse.metadataLocation(); - } else if (response instanceof IcebergCatalogHandler.StringLoadTableResponse stringResponse) { - metadataLocation = stringResponse.metadataLocation(); } else { throw new IllegalStateException("Cannot build etag from: " + response); } @@ -587,9 +585,6 @@ public Response loadCredentials( final List credentials; if (restResponse instanceof LoadTableResponse loadTableResponse) { credentials = loadTableResponse.credentials(); - } else if (restResponse - instanceof IcebergCatalogHandler.StringLoadTableResponse stringResponse) { - credentials = stringResponse.credentials(); } else { throw new IllegalStateException("Cannot extract credentials from " + restResponse); } diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index f98de937fb..93d13f42e3 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -18,10 +18,9 @@ */ package org.apache.polaris.service.catalog.iceberg; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonRawValue; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; +import jakarta.annotation.Nonnull; import jakarta.ws.rs.core.SecurityContext; import java.io.Closeable; import java.time.OffsetDateTime; @@ -38,10 +37,10 @@ import org.apache.iceberg.BaseTable; import org.apache.iceberg.MetadataUpdate; import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.SnapshotRef; import org.apache.iceberg.SortOrder; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; -import org.apache.iceberg.TableMetadataParser; import org.apache.iceberg.TableOperations; import org.apache.iceberg.UpdateRequirement; import org.apache.iceberg.catalog.Catalog; @@ -59,7 +58,6 @@ import org.apache.iceberg.rest.HTTPClient; import org.apache.iceberg.rest.RESTCatalog; import org.apache.iceberg.rest.RESTResponse; -import org.apache.iceberg.rest.credentials.Credential; import org.apache.iceberg.rest.credentials.ImmutableCredential; import org.apache.iceberg.rest.requests.CommitTransactionRequest; import org.apache.iceberg.rest.requests.CreateNamespaceRequest; @@ -104,7 +102,6 @@ import org.apache.polaris.service.context.catalog.CallContextCatalogFactory; import org.apache.polaris.service.http.IcebergHttpUtil; import org.apache.polaris.service.http.IfNoneMatch; -import org.apache.polaris.service.persistence.MetadataJson; import org.apache.polaris.service.types.NotificationRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -444,16 +441,10 @@ public RESTResponse createTableDirectWithWriteDelegation( .withProperties(properties) .create(); if (table instanceof BaseTable baseTable) { - final MetadataJson tableMetadataJson; - if (baseCatalog instanceof IcebergCatalog icebergCatalog) { - tableMetadataJson = icebergCatalog.loadTableMetadataJson(tableIdentifier); - } else { - TableMetadata tableMetadata = baseTable.operations().current(); - tableMetadataJson = MetadataJson.fromMetadata(tableMetadata); - } + TableMetadata tableMetadata = baseTable.operations().current(); return buildLoadTableResponseWithDelegationCredentials( tableIdentifier, - tableMetadataJson, + tableMetadata, Set.of( PolarisStorageActions.READ, PolarisStorageActions.WRITE, PolarisStorageActions.LIST)); } else if (table instanceof BaseMetadataTable) { @@ -544,7 +535,7 @@ public RESTResponse createTableStagedWithWriteDelegation( TableMetadata metadata = stageTableCreateHelper(namespace, request); return buildLoadTableResponseWithDelegationCredentials( - ident, MetadataJson.fromMetadata(metadata), Set.of(PolarisStorageActions.ALL)); + ident, metadata, Set.of(PolarisStorageActions.ALL)); } /** @@ -651,16 +642,14 @@ public Optional loadTableIfStale( } } + LoadTableResponse rawResponse; if (baseCatalog instanceof IcebergCatalog icebergCatalog) { - MetadataJson metadataJson = icebergCatalog.loadTableMetadataJson(tableIdentifier); - if (icebergCatalog.shouldFilterSnapshots(snapshots)) { - TableMetadata metadata = TableMetadataParser.fromJson(metadataJson.content()); - metadataJson = MetadataJson.fromMetadata(metadata, snapshots); // round-trip to filter - } - return Optional.of(new StringLoadTableResponse(metadataJson)); + TableMetadata metadata = icebergCatalog.loadTableMetadata(tableIdentifier); + rawResponse = LoadTableResponse.builder().withTableMetadata(metadata).build(); } else { - return Optional.of(catalogHandlerUtils.loadTable(baseCatalog, tableIdentifier)); + rawResponse = catalogHandlerUtils.loadTable(baseCatalog, tableIdentifier); } + return Optional.of(filterResponseToSnapshots(rawResponse, snapshots)); } public RESTResponse loadTableWithAccessDelegation( @@ -750,48 +739,48 @@ public Optional loadTableWithAccessDelegationIfStale( } } } - MetadataJson metadataJson = null; + TableMetadata metadata = null; if (baseCatalog instanceof IcebergCatalog icebergCatalog) { - metadataJson = icebergCatalog.loadTableMetadataJson(tableIdentifier, snapshots); - if (icebergCatalog.shouldFilterSnapshots(snapshots)) { - TableMetadata metadata = TableMetadataParser.fromJson(metadataJson.content()); - metadataJson = MetadataJson.fromMetadata(metadata, snapshots); // round-trip to filter - } + metadata = icebergCatalog.loadTableMetadata(tableIdentifier); } // The metadata failed to load - if (metadataJson == null) { + if (metadata == null) { throw new NoSuchTableException("Table does not exist: %s", tableIdentifier.toString()); } else { + metadata = filterMetadataToSnapshots(metadata, snapshots); return Optional.of( buildLoadTableResponseWithDelegationCredentials( - tableIdentifier, metadataJson, actionsRequested)); + tableIdentifier, metadata, actionsRequested)); } } private RESTResponse buildLoadTableResponseWithDelegationCredentials( TableIdentifier tableIdentifier, - MetadataJson metadataJson, + TableMetadata tableMetadata, Set actions) { - Map config = Map.of(); - List credentials = List.of(); + LoadTableResponse.Builder responseBuilder = + LoadTableResponse.builder().withTableMetadata(tableMetadata); if (baseCatalog instanceof SupportsCredentialDelegation credentialDelegation) { LOGGER .atDebug() .addKeyValue("tableIdentifier", tableIdentifier) - .addKeyValue("tableLocation", metadataJson.location()) + .addKeyValue("tableLocation", tableMetadata.location()) .log("Fetching client credentials for table"); AccessConfig accessConfig = - credentialDelegation.getAccessConfig( - tableIdentifier, metadataJson.tableLocations(), actions); - config = accessConfig.credentials(); - if (!config.isEmpty()) { - credentials.add( - ImmutableCredential.builder().prefix(metadataJson.location()).config(config).build()); + credentialDelegation.getAccessConfig(tableIdentifier, tableMetadata, actions); + Map credentialConfig = accessConfig.credentials(); + responseBuilder.addAllConfig(credentialConfig); + responseBuilder.addAllConfig(accessConfig.extraProperties()); + if (!credentialConfig.isEmpty()) { + responseBuilder.addCredential( + ImmutableCredential.builder() + .prefix(tableMetadata.location()) + .config(credentialConfig) + .build()); } } - return new StringLoadTableResponse( - metadataJson.location(), metadataJson.content(), config, credentials); + return responseBuilder.build(); } private UpdateTableRequest applyUpdateFilters(UpdateTableRequest request) { @@ -886,7 +875,7 @@ public void tableExists(TableIdentifier tableIdentifier) { op, PolarisEntitySubType.ICEBERG_TABLE, tableIdentifier); if (baseCatalog instanceof IcebergCatalog icebergCatalog) { - icebergCatalog.loadTableMetadataJson(tableIdentifier); + icebergCatalog.loadTableMetadata(tableIdentifier); } else { catalogHandlerUtils.loadTable(baseCatalog, tableIdentifier); } @@ -952,16 +941,12 @@ public void commitTransaction(CommitTransactionRequest commitTransactionRequest) final TableOperations tableOps; if (baseCatalog instanceof IcebergCatalog icebergCatalog) { tableOps = icebergCatalog.newTableOps(change.identifier()); - MetadataJson metadataJson = - icebergCatalog.loadTableMetadataJson(change.identifier()); - currentMetadata = - TableMetadata.buildFrom(TableMetadataParser.fromJson(metadataJson.content())) - .withMetadataLocation(metadataJson.location()) - .build(); + currentMetadata = icebergCatalog.loadTableMetadata(change.identifier()); // Update tableOps.current() to reflect the cached metadata - if (tableOps instanceof IcebergCatalog.BasePolarisTableOperations bpto) { - bpto.setCurrentMetadata(metadataJson.location(), currentMetadata); - bpto.shouldRefresh = false; + if (tableOps instanceof IcebergCatalog.BasePolarisTableOperations polarisOps) { + polarisOps.setCurrentMetadata( + currentMetadata.metadataFileLocation(), currentMetadata); + polarisOps.shouldRefresh = false; } } else { final Table table = baseCatalog.loadTable(change.identifier()); @@ -1132,26 +1117,34 @@ public void renameView(RenameTableRequest request) { catalogHandlerUtils.renameView(viewCatalog, request); } - @Override - public void close() throws Exception { - if (baseCatalog instanceof Closeable closeable) { - closeable.close(); - } + private @Nonnull TableMetadata filterMetadataToSnapshots( + TableMetadata metadata, String snapshots) { + Set referencedSnapshotIds = + metadata.refs().values().stream().map(SnapshotRef::snapshotId).collect(Collectors.toSet()); + return metadata.removeSnapshotsIf(s -> !referencedSnapshotIds.contains(s.snapshotId())); } - public record StringLoadTableResponse( - @JsonProperty("metadata-location") String metadataLocation, - @JsonProperty("metadata") @JsonRawValue String metadata, - @JsonProperty("config") Map config, - @JsonProperty("storage-credentials") List credentials) - implements RESTResponse { - @Override - public void validate() { - Preconditions.checkNotNull(this.metadata, "Invalid metadata: null"); + private @Nonnull LoadTableResponse filterResponseToSnapshots( + LoadTableResponse loadTableResponse, String snapshots) { + if (snapshots == null || snapshots.equalsIgnoreCase(SNAPSHOTS_ALL)) { + return loadTableResponse; + } else if (snapshots.equalsIgnoreCase(SNAPSHOTS_REFS)) { + TableMetadata filteredMetadata = + filterMetadataToSnapshots(loadTableResponse.tableMetadata(), snapshots); + return LoadTableResponse.builder() + .withTableMetadata(filteredMetadata) + .addAllConfig(loadTableResponse.config()) + .addAllCredentials(loadTableResponse.credentials()) + .build(); + } else { + throw new IllegalArgumentException("Unrecognized snapshots: " + snapshots); } + } - public StringLoadTableResponse(MetadataJson metadataJson) { - this(metadataJson.location(), metadataJson.content(), Map.of(), List.of()); + @Override + public void close() throws Exception { + if (baseCatalog instanceof Closeable closeable) { + closeable.close(); } } } diff --git a/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index d48b43a043..81d48a2309 100644 --- a/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -19,16 +19,13 @@ package org.apache.polaris.service.persistence; import java.util.Map; -import java.util.Objects; import java.util.function.Supplier; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.entity.PolarisEntity; -import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.table.IcebergTableLikeEntity; @@ -51,7 +48,7 @@ public class MetadataCacheManager { * Load the cached metadata.json content and location or fall back to `fallback` if one doesn't * exist. If the metadata is not currently cached, it may be added to the cache. */ - public static MetadataJson loadTableMetadataJson( + public static TableMetadata loadTableMetadata( TableIdentifier tableIdentifier, int maxBytesToCache, PolarisCallContext callContext, @@ -63,32 +60,27 @@ public static MetadataJson loadTableMetadataJson( tableIdentifier, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ICEBERG_TABLE); // If the table doesn't exist, just fall back fast if (resolvedEntities == null) { - return MetadataJson.fromMetadata(fallback.get()); + return fallback.get(); } LOGGER.debug(String.format("Loading cached metadata for %s", tableIdentifier)); IcebergTableLikeEntity tableLikeEntity = IcebergTableLikeEntity.of(resolvedEntities.getRawLeafEntity()); - String cacheContent = tableLikeEntity.getMetadataCacheContent(); - if (cacheContent != null) { + Map tableEntityProperties = tableLikeEntity.getInternalPropertiesAsMap(); + String cacheContent = + tableEntityProperties.get(IcebergTableLikeEntity.METADATA_CACHE_CONTENT_KEY); + if (cacheContent != null && !cacheContent.isEmpty()) { LOGGER.debug(String.format("Using cached metadata for %s", tableIdentifier)); - Map entityProperties = tableLikeEntity.getPropertiesAsMap(); - return new MetadataJson( - tableLikeEntity.getMetadataLocation(), - tableLikeEntity.getMetadataCacheContent(), - Stream.of( - entityProperties.get(PolarisEntityConstants.ENTITY_BASE_LOCATION), - entityProperties.get( - IcebergTableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY), - entityProperties.get( - IcebergTableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY)) - .filter(Objects::nonNull) - .collect(Collectors.toSet())); + TableMetadata tableMetadata = TableMetadataParser.fromJson(cacheContent); + return TableMetadata.buildFrom(tableMetadata) + .withMetadataLocation( + tableEntityProperties.get(IcebergTableLikeEntity.METADATA_LOCATION_KEY)) + .build(); } else { - MetadataJson fallbackJson = MetadataJson.fromMetadata(fallback.get()); + TableMetadata fallbackMetadata = fallback.get(); var cacheResult = - cacheTableMetadataJson( + cacheTableMetadata( tableLikeEntity, - fallbackJson.content(), + fallbackMetadata, maxBytesToCache, callContext, metastoreManager, @@ -96,7 +88,7 @@ public static MetadataJson loadTableMetadataJson( if (!cacheResult.isSuccess()) { LOGGER.debug(String.format("Failed to cache metadata for %s", tableIdentifier)); } - return fallbackJson; + return fallbackMetadata; } } @@ -105,15 +97,16 @@ public static MetadataJson loadTableMetadataJson( * * @return The result of trying to cache the metadata */ - private static EntityResult cacheTableMetadataJson( + private static EntityResult cacheTableMetadata( IcebergTableLikeEntity tableLikeEntity, - String metadataJson, + TableMetadata tableMetadata, int maxBytesToCache, PolarisCallContext callContext, PolarisMetaStoreManager metaStoreManager, PolarisResolutionManifestCatalogView resolvedEntityView) { + String metadataString = TableMetadataParser.toJson(tableMetadata); if (maxBytesToCache != FeatureConfiguration.METADATA_CACHE_MAX_BYTES_INFINITE_CACHING) { - if (metadataJson.length() > maxBytesToCache) { + if (metadataString.length() * 2 > maxBytesToCache) { LOGGER.debug( String.format( "Will not cache metadata for %s; metadata above the limit of %d bytes", @@ -125,7 +118,7 @@ private static EntityResult cacheTableMetadataJson( LOGGER.debug(String.format("Caching metadata for %s", tableLikeEntity.getTableIdentifier())); TableLikeEntity newTableLikeEntity = new IcebergTableLikeEntity.Builder(tableLikeEntity) - .setMetadataContent(tableLikeEntity.getMetadataLocation(), metadataJson) + .setMetadataContent(tableLikeEntity.getMetadataLocation(), metadataString) .build(); PolarisResolvedPathWrapper resolvedPath = resolvedEntityView.getResolvedPath( diff --git a/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataJson.java b/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataJson.java deleted file mode 100644 index ae0ce858f7..0000000000 --- a/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataJson.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.persistence; - -import static org.apache.polaris.service.catalog.iceberg.IcebergCatalogHandler.SNAPSHOTS_ALL; -import static org.apache.polaris.service.catalog.iceberg.IcebergCatalogHandler.SNAPSHOTS_REFS; - -import java.util.Set; -import java.util.stream.Collectors; -import org.apache.iceberg.SnapshotRef; -import org.apache.iceberg.TableMetadata; -import org.apache.iceberg.TableMetadataParser; -import org.apache.polaris.service.catalog.iceberg.IcebergMetadataUtil; - -/** - * Represents a metadata.json file with its location and content - * - * @param location the location of the metadata.json itself - * @param content the content of the metadata.json - * @param tableLocations The locations that the table this metadata.json describes might be written. - * This includes the table base location as well as any data or metadata locations specified in - * the table properties - */ -public record MetadataJson(String location, String content, Set tableLocations) { - - /** Construct a record from a {@link TableMetadata} object */ - public static MetadataJson fromMetadata(TableMetadata metadata) { - return fromMetadata(metadata, SNAPSHOTS_ALL); - } - - /** See {@link MetadataJson#fromMetadata(TableMetadata)} */ - public static MetadataJson fromMetadata(TableMetadata metadata, String snapshots) { - final TableMetadata filteredMetadata; - if (snapshots != null && !snapshots.equalsIgnoreCase(SNAPSHOTS_ALL)) { - if (snapshots.equalsIgnoreCase(SNAPSHOTS_REFS)) { - Set referencedSnapshotIds = - metadata.refs().values().stream() - .map(SnapshotRef::snapshotId) - .collect(Collectors.toSet()); - - filteredMetadata = - metadata.removeSnapshotsIf(s -> !referencedSnapshotIds.contains(s.snapshotId())); - } else { - throw new IllegalArgumentException("Unrecognized snapshots: " + snapshots); - } - } else { - filteredMetadata = metadata; - } - Set tableLocations = - IcebergMetadataUtil.getLocationsAllowedToBeAccessed(filteredMetadata); - return new MetadataJson( - metadata.metadataFileLocation(), - TableMetadataParser.toJson(filteredMetadata), - tableLocations); - } -} From a7d5ebd181649947f65f7beeca1b735c24cf9a6e Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 16 May 2025 13:26:04 -0700 Subject: [PATCH 81/88] constnats --- .../apache/polaris/core/config/FeatureConfiguration.java | 9 ++++++--- .../polaris/service/catalog/iceberg/IcebergCatalog.java | 6 +++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java index fd8d4da07d..84afbc2553 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java @@ -44,6 +44,11 @@ protected FeatureConfiguration( super(key, description, defaultValue, catalogConfig, catalogConfigUnsafe, validation); } + public static final class Constants { + public static final int METADATA_CACHE_MAX_BYTES_NO_CACHING = 0; + public static final int METADATA_CACHE_MAX_BYTES_INFINITE_CACHING = -1; + } + /** * Helper for the common scenario of gating a feature with a boolean FeatureConfiguration, where * we want to throw an UnsupportedOperationException if it's not enabled. @@ -272,8 +277,6 @@ public static void enforceFeatureEnabledOrThrow( .defaultValue(4) .buildFeatureConfiguration(); - public static final int METADATA_CACHE_MAX_BYTES_NO_CACHING = 0; - public static final int METADATA_CACHE_MAX_BYTES_INFINITE_CACHING = -1; public static final PolarisConfiguration METADATA_CACHE_MAX_BYTES = PolarisConfiguration.builder() .key("METADATA_CACHE_MAX_BYTES") @@ -282,7 +285,7 @@ public static void enforceFeatureEnabledOrThrow( "If nonzero, the approximate max size a table's metadata can be in order to be cached in the persistence" + " layer. If zero, no metadata will be cached or served from the cache. If -1, all metadata" + " will be cached.") - .defaultValue(METADATA_CACHE_MAX_BYTES_NO_CACHING) + .defaultValue(Constants.METADATA_CACHE_MAX_BYTES_NO_CACHING) .validation(value -> value >= -1) .buildFeatureConfiguration(); diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index e18c472d06..76834c5b8b 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -919,7 +919,7 @@ public TableMetadata loadTableMetadata(TableIdentifier identifier) { () -> { return loadTableMetadata(loadTable(identifier)); }; - if (maxMetadataCacheBytes == FeatureConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING) { + if (maxMetadataCacheBytes == FeatureConfiguration.Constants.METADATA_CACHE_MAX_BYTES_NO_CACHING) { return fallback.get(); } else { return MetadataCacheManager.loadTableMetadata( @@ -1514,8 +1514,8 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { FeatureConfiguration.METADATA_CACHE_MAX_BYTES); Optional metadataJsonToCache = switch (maxMetadataCacheBytes) { - case FeatureConfiguration.METADATA_CACHE_MAX_BYTES_NO_CACHING -> Optional.empty(); - case FeatureConfiguration.METADATA_CACHE_MAX_BYTES_INFINITE_CACHING -> { + case FeatureConfiguration.Constants.METADATA_CACHE_MAX_BYTES_NO_CACHING -> Optional.empty(); + case FeatureConfiguration.Constants.METADATA_CACHE_MAX_BYTES_INFINITE_CACHING -> { yield Optional.of(TableMetadataParser.toJson(metadata)); } default -> { From 1ee726058c3a1a7554edf435ef636a982ae0366f Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 16 May 2025 13:28:59 -0700 Subject: [PATCH 82/88] polish --- .../quarkus/catalog/IcebergCatalogTest.java | 25 ++++++++++--------- .../catalog/iceberg/IcebergCatalog.java | 6 +++-- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java index 0279f30235..4fee06fa55 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java @@ -302,21 +302,22 @@ public void before(TestInfo testInfo) { .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) .setAllowedLocations(List.of(storageLocation, "s3://externally-owned-bucket")) .build(); - CatalogEntity catalogEntityWithProperties = - new CatalogEntity.Builder() - .setName(CATALOG_NAME) - .setDefaultBaseLocation(storageLocation) - .setReplaceNewLocationPrefixWithCatalogDefault("file:") - .addProperty(FeatureConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") - .addProperty( - FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .addProperty(FeatureConfiguration.METADATA_CACHE_MAX_BYTES.catalogConfig(), "-1") - .setStorageConfigurationInfo(storageConfigModel, storageLocation) - .build(); catalogEntity = adminService.createCatalog( - new CreateCatalogRequest(catalogEntityWithProperties.asCatalog())); + new CreateCatalogRequest( + new CatalogEntity.Builder() + .setName(CATALOG_NAME) + .setDefaultBaseLocation(storageLocation) + .setReplaceNewLocationPrefixWithCatalogDefault("file:") + .addProperty( + FeatureConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") + .addProperty( + FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), + "true") + .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .build() + .asCatalog())); passthroughView = new PolarisPassthroughResolutionView( diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index 76834c5b8b..7019dc773f 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -919,7 +919,8 @@ public TableMetadata loadTableMetadata(TableIdentifier identifier) { () -> { return loadTableMetadata(loadTable(identifier)); }; - if (maxMetadataCacheBytes == FeatureConfiguration.Constants.METADATA_CACHE_MAX_BYTES_NO_CACHING) { + if (maxMetadataCacheBytes + == FeatureConfiguration.Constants.METADATA_CACHE_MAX_BYTES_NO_CACHING) { return fallback.get(); } else { return MetadataCacheManager.loadTableMetadata( @@ -1514,7 +1515,8 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { FeatureConfiguration.METADATA_CACHE_MAX_BYTES); Optional metadataJsonToCache = switch (maxMetadataCacheBytes) { - case FeatureConfiguration.Constants.METADATA_CACHE_MAX_BYTES_NO_CACHING -> Optional.empty(); + case FeatureConfiguration.Constants.METADATA_CACHE_MAX_BYTES_NO_CACHING -> + Optional.empty(); case FeatureConfiguration.Constants.METADATA_CACHE_MAX_BYTES_INFINITE_CACHING -> { yield Optional.of(TableMetadataParser.toJson(metadata)); } From 35fb80941d02b108eff57018d2bd875eb4497bd1 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 16 May 2025 13:30:41 -0700 Subject: [PATCH 83/88] another change --- .../apache/polaris/core/persistence/cache/EntityWeigher.java | 2 +- .../apache/polaris/service/catalog/iceberg/IcebergCatalog.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java index b3409d3b90..3677984366 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityWeigher.java @@ -35,7 +35,7 @@ public class EntityWeigher implements Weigher { private static final int APPROXIMATE_ENTITY_OVERHEAD = 1000; /* Represents the amount of bytes that a character is expected to take up */ - private static final int APPROXIMATE_BYTES_PER_CHAR = 3; + public static final int APPROXIMATE_BYTES_PER_CHAR = 3; /** Singleton instance */ private static final EntityWeigher instance = new EntityWeigher(); diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index 7019dc773f..b14adf780d 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -105,6 +105,7 @@ import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; import org.apache.polaris.core.persistence.ResolvedPolarisEntity; +import org.apache.polaris.core.persistence.cache.EntityWeigher; import org.apache.polaris.core.persistence.dao.entity.BaseResult; import org.apache.polaris.core.persistence.dao.entity.DropEntityResult; import org.apache.polaris.core.persistence.dao.entity.EntityResult; @@ -1522,7 +1523,7 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { } default -> { String rawMetadataJson = TableMetadataParser.toJson(metadata); - if (rawMetadataJson.length() * 2 < maxMetadataCacheBytes) { + if (rawMetadataJson.length() * EntityWeigher.APPROXIMATE_BYTES_PER_CHAR < maxMetadataCacheBytes) { yield Optional.of(rawMetadataJson); } else { yield Optional.empty(); From 6d8d70342418ba3782738fede49fc6a57585706a Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 16 May 2025 13:30:44 -0700 Subject: [PATCH 84/88] autolint --- .../apache/polaris/service/catalog/iceberg/IcebergCatalog.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index b14adf780d..df06426d4c 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -1523,7 +1523,8 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { } default -> { String rawMetadataJson = TableMetadataParser.toJson(metadata); - if (rawMetadataJson.length() * EntityWeigher.APPROXIMATE_BYTES_PER_CHAR < maxMetadataCacheBytes) { + if (rawMetadataJson.length() * EntityWeigher.APPROXIMATE_BYTES_PER_CHAR + < maxMetadataCacheBytes) { yield Optional.of(rawMetadataJson); } else { yield Optional.empty(); From eb229015aa070ff1d160e6f39b08d5b0711c8b5e Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 16 May 2025 14:51:28 -0700 Subject: [PATCH 85/88] fixes --- .../core/entity/table/IcebergTableLikeEntity.java | 2 +- .../service/quarkus/catalog/IcebergCatalogTest.java | 2 +- .../service/persistence/MetadataCacheManager.java | 13 +++++++------ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/table/IcebergTableLikeEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/table/IcebergTableLikeEntity.java index 6d48a55051..9372e25700 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/table/IcebergTableLikeEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/table/IcebergTableLikeEntity.java @@ -45,7 +45,7 @@ public class IcebergTableLikeEntity extends TableLikeEntity { // For applicable types, this key on the "internalProperties" map will return the location of the // `metadata.json` that is cached in `METADATA_CACHE_CONTENT_KEY`. This will often match the // current metadata location in `METADATA_LOCATION_KEY`; if it does not the cache is invalid - private static final String METADATA_CACHE_LOCATION_KEY = "metadata-cache-location"; + public static final String METADATA_CACHE_LOCATION_KEY = "metadata-cache-location"; public static final String USER_SPECIFIED_WRITE_DATA_LOCATION_KEY = "write.data.path"; public static final String USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY = "write.metadata.path"; diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java index 4fee06fa55..64acc63164 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java @@ -183,7 +183,7 @@ public Map getConfigOverrides() { "polaris.event-listener.type", "test", "polaris.features." + FeatureConfiguration.METADATA_CACHE_MAX_BYTES.key, - "10000"); + String.valueOf(FeatureConfiguration.Constants.METADATA_CACHE_MAX_BYTES_INFINITE_CACHING)); } } diff --git a/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java b/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java index 81d48a2309..664f659b36 100644 --- a/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java +++ b/service/common/src/main/java/org/apache/polaris/service/persistence/MetadataCacheManager.java @@ -66,15 +66,15 @@ public static TableMetadata loadTableMetadata( IcebergTableLikeEntity tableLikeEntity = IcebergTableLikeEntity.of(resolvedEntities.getRawLeafEntity()); Map tableEntityProperties = tableLikeEntity.getInternalPropertiesAsMap(); + String entityLocation = tableEntityProperties.get(IcebergTableLikeEntity.METADATA_LOCATION_KEY); String cacheContent = tableEntityProperties.get(IcebergTableLikeEntity.METADATA_CACHE_CONTENT_KEY); - if (cacheContent != null && !cacheContent.isEmpty()) { + String cacheLocation = + tableEntityProperties.get(IcebergTableLikeEntity.METADATA_CACHE_LOCATION_KEY); + if (cacheContent != null && !cacheContent.isEmpty() && entityLocation.equals(cacheLocation)) { LOGGER.debug(String.format("Using cached metadata for %s", tableIdentifier)); TableMetadata tableMetadata = TableMetadataParser.fromJson(cacheContent); - return TableMetadata.buildFrom(tableMetadata) - .withMetadataLocation( - tableEntityProperties.get(IcebergTableLikeEntity.METADATA_LOCATION_KEY)) - .build(); + return TableMetadata.buildFrom(tableMetadata).withMetadataLocation(cacheLocation).build(); } else { TableMetadata fallbackMetadata = fallback.get(); var cacheResult = @@ -105,7 +105,8 @@ private static EntityResult cacheTableMetadata( PolarisMetaStoreManager metaStoreManager, PolarisResolutionManifestCatalogView resolvedEntityView) { String metadataString = TableMetadataParser.toJson(tableMetadata); - if (maxBytesToCache != FeatureConfiguration.METADATA_CACHE_MAX_BYTES_INFINITE_CACHING) { + if (maxBytesToCache + != FeatureConfiguration.Constants.METADATA_CACHE_MAX_BYTES_INFINITE_CACHING) { if (metadataString.length() * 2 > maxBytesToCache) { LOGGER.debug( String.format( From 0f4125d0aeed9c014f8921232a2806b7c79449d0 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 22 May 2025 15:25:09 -0700 Subject: [PATCH 86/88] comments --- .../service/catalog/iceberg/IcebergCatalog.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index df06426d4c..4ecfb448dc 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -1327,14 +1327,12 @@ public void commit(TableMetadata base, TableMetadata metadata) { if (base == metadata) { LOGGER.info("Nothing to commit."); return; - } else { - if (base != null + } else if (base != null && metadata != null && base.metadataFileLocation().equals(metadata.metadataFileLocation())) { - // if the metadata is not changed, return early - LOGGER.info("Nothing to commit."); - return; - } + // if the metadata is not changed, return early + LOGGER.info("Nothing to commit."); + return; } long start = System.currentTimeMillis(); @@ -1516,8 +1514,9 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { FeatureConfiguration.METADATA_CACHE_MAX_BYTES); Optional metadataJsonToCache = switch (maxMetadataCacheBytes) { - case FeatureConfiguration.Constants.METADATA_CACHE_MAX_BYTES_NO_CACHING -> - Optional.empty(); + case FeatureConfiguration.Constants.METADATA_CACHE_MAX_BYTES_NO_CACHING -> { + yield Optional.empty(); + } case FeatureConfiguration.Constants.METADATA_CACHE_MAX_BYTES_INFINITE_CACHING -> { yield Optional.of(TableMetadataParser.toJson(metadata)); } From ca85168c26e02236d721c8cbbf75bdef142b25cb Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 22 May 2025 15:26:56 -0700 Subject: [PATCH 87/88] another comment --- .../iceberg/IcebergCatalogHandler.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index 93d13f42e3..0e680dbb77 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -1119,28 +1119,28 @@ public void renameView(RenameTableRequest request) { private @Nonnull TableMetadata filterMetadataToSnapshots( TableMetadata metadata, String snapshots) { - Set referencedSnapshotIds = - metadata.refs().values().stream().map(SnapshotRef::snapshotId).collect(Collectors.toSet()); - return metadata.removeSnapshotsIf(s -> !referencedSnapshotIds.contains(s.snapshotId())); - } - - private @Nonnull LoadTableResponse filterResponseToSnapshots( - LoadTableResponse loadTableResponse, String snapshots) { if (snapshots == null || snapshots.equalsIgnoreCase(SNAPSHOTS_ALL)) { - return loadTableResponse; + return metadata; } else if (snapshots.equalsIgnoreCase(SNAPSHOTS_REFS)) { - TableMetadata filteredMetadata = - filterMetadataToSnapshots(loadTableResponse.tableMetadata(), snapshots); - return LoadTableResponse.builder() - .withTableMetadata(filteredMetadata) - .addAllConfig(loadTableResponse.config()) - .addAllCredentials(loadTableResponse.credentials()) - .build(); + Set referencedSnapshotIds = + metadata.refs().values().stream().map(SnapshotRef::snapshotId).collect(Collectors.toSet()); + return metadata.removeSnapshotsIf(s -> !referencedSnapshotIds.contains(s.snapshotId())); } else { throw new IllegalArgumentException("Unrecognized snapshots: " + snapshots); } } + private @Nonnull LoadTableResponse filterResponseToSnapshots( + LoadTableResponse loadTableResponse, String snapshots) { + TableMetadata filteredMetadata = + filterMetadataToSnapshots(loadTableResponse.tableMetadata(), snapshots); + return LoadTableResponse.builder() + .withTableMetadata(filteredMetadata) + .addAllConfig(loadTableResponse.config()) + .addAllCredentials(loadTableResponse.credentials()) + .build(); + } + @Override public void close() throws Exception { if (baseCatalog instanceof Closeable closeable) { From 17b950ff6947927a9972e9ca1d9c405129b959bb Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 22 May 2025 15:34:01 -0700 Subject: [PATCH 88/88] autolint --- .../polaris/service/catalog/iceberg/IcebergCatalog.java | 4 ++-- .../service/catalog/iceberg/IcebergCatalogHandler.java | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index 6d47d4711b..997874fe0f 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -1288,8 +1288,8 @@ public void commit(TableMetadata base, TableMetadata metadata) { LOGGER.info("Nothing to commit."); return; } else if (base != null - && metadata != null - && base.metadataFileLocation().equals(metadata.metadataFileLocation())) { + && metadata != null + && base.metadataFileLocation().equals(metadata.metadataFileLocation())) { // if the metadata is not changed, return early LOGGER.info("Nothing to commit."); return; diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index 0e680dbb77..ced5545b1c 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -1123,7 +1123,9 @@ public void renameView(RenameTableRequest request) { return metadata; } else if (snapshots.equalsIgnoreCase(SNAPSHOTS_REFS)) { Set referencedSnapshotIds = - metadata.refs().values().stream().map(SnapshotRef::snapshotId).collect(Collectors.toSet()); + metadata.refs().values().stream() + .map(SnapshotRef::snapshotId) + .collect(Collectors.toSet()); return metadata.removeSnapshotsIf(s -> !referencedSnapshotIds.contains(s.snapshotId())); } else { throw new IllegalArgumentException("Unrecognized snapshots: " + snapshots);