diff --git a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java index f19194f33d..b6f60164ed 100644 --- a/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java +++ b/extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java @@ -197,21 +197,23 @@ public synchronized Supplier getOrCreateSessionSupplier( @Override public synchronized StorageCredentialCache getOrCreateStorageCredentialCache( - RealmContext realmContext) { + RealmContext realmContext, PolarisCallContext polarisCallContext) { if (!storageCredentialCacheMap.containsKey(realmContext.getRealmIdentifier())) { storageCredentialCacheMap.put( - realmContext.getRealmIdentifier(), new StorageCredentialCache()); + realmContext.getRealmIdentifier(), new StorageCredentialCache(polarisCallContext)); } return storageCredentialCacheMap.get(realmContext.getRealmIdentifier()); } @Override - public synchronized EntityCache getOrCreateEntityCache(RealmContext realmContext) { + public synchronized EntityCache getOrCreateEntityCache( + RealmContext realmContext, PolarisCallContext polarisCallContext) { if (!entityCacheMap.containsKey(realmContext.getRealmIdentifier())) { PolarisMetaStoreManager metaStoreManager = getOrCreateMetaStoreManager(realmContext); entityCacheMap.put( - realmContext.getRealmIdentifier(), new InMemoryEntityCache(metaStoreManager)); + realmContext.getRealmIdentifier(), + new InMemoryEntityCache(metaStoreManager, polarisCallContext)); } return entityCacheMap.get(realmContext.getRealmIdentifier()); diff --git a/polaris-core/build.gradle.kts b/polaris-core/build.gradle.kts index 340dfc472e..7a7957d6f0 100644 --- a/polaris-core/build.gradle.kts +++ b/polaris-core/build.gradle.kts @@ -36,6 +36,7 @@ dependencies { implementation("com.fasterxml.jackson.core:jackson-annotations") implementation("com.fasterxml.jackson.core:jackson-core") implementation("com.fasterxml.jackson.core:jackson-databind") + implementation(libs.jakarta.enterprise.cdi.api) implementation(libs.caffeine) implementation(libs.commons.lang3) implementation(libs.commons.codec1) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/PolarisConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/PolarisConfiguration.java index bcb3809881..f997ff7e5b 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/PolarisConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/PolarisConfiguration.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; -import org.apache.polaris.core.context.CallContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -116,25 +115,6 @@ public BehaviorChangeConfiguration buildBehaviorChangeConfiguration() { } } - /** - * Returns the value of a `PolarisConfiguration`, or the default if it cannot be loaded. This - * method does not need to be used when a `CallContext` is already available - */ - public static T loadConfig(PolarisConfiguration configuration) { - var callContext = CallContext.getCurrentContext(); - if (callContext == null) { - LOGGER.warn( - String.format( - "Unable to load current call context; using %s = %s", - configuration.key, configuration.defaultValue)); - return configuration.defaultValue; - } - return callContext - .getPolarisCallContext() - .getConfigurationStore() - .getConfiguration(callContext.getPolarisCallContext(), configuration); - } - public static Builder builder() { return new Builder<>(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java index a995883304..8a8f42bf4b 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java @@ -30,6 +30,7 @@ import java.util.Optional; import java.util.Set; import org.apache.iceberg.exceptions.BadRequestException; +import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; import org.apache.polaris.core.admin.model.AzureStorageConfigInfo; @@ -80,7 +81,7 @@ public static CatalogEntity of(PolarisBaseEntity sourceEntity) { return null; } - public static CatalogEntity fromCatalog(Catalog catalog) { + public static CatalogEntity fromCatalog(PolarisCallContext callContext, Catalog catalog) { Builder builder = new Builder() .setName(catalog.getName()) @@ -90,7 +91,7 @@ public static CatalogEntity fromCatalog(Catalog catalog) { internalProperties.put(CATALOG_TYPE_PROPERTY, catalog.getType().name()); builder.setInternalProperties(internalProperties); builder.setStorageConfigurationInfo( - catalog.getStorageConfigInfo(), getDefaultBaseLocation(catalog)); + catalog.getStorageConfigInfo(), getDefaultBaseLocation(catalog), callContext); return builder.build(); } @@ -247,7 +248,9 @@ public Builder setReplaceNewLocationPrefixWithCatalogDefault(String value) { } public Builder setStorageConfigurationInfo( - StorageConfigInfo storageConfigModel, String defaultBaseLocation) { + StorageConfigInfo storageConfigModel, + String defaultBaseLocation, + PolarisCallContext callContext) { if (storageConfigModel != null) { PolarisStorageConfigurationInfo config; Set allowedLocations = new HashSet<>(storageConfigModel.getAllowedLocations()); @@ -261,7 +264,7 @@ public Builder setStorageConfigurationInfo( throw new BadRequestException("Must specify default base location"); } allowedLocations.add(defaultBaseLocation); - validateMaxAllowedLocations(allowedLocations); + validateMaxAllowedLocations(allowedLocations, callContext); switch (storageConfigModel.getStorageType()) { case S3: AwsStorageConfigInfo awsConfigModel = (AwsStorageConfigInfo) storageConfigModel; @@ -305,10 +308,13 @@ public Builder setStorageConfigurationInfo( } /** Validate the number of allowed locations not exceeding the max value. */ - private void validateMaxAllowedLocations(Collection allowedLocations) { + private void validateMaxAllowedLocations( + Collection allowedLocations, PolarisCallContext callContext) { int maxAllowedLocations = - BehaviorChangeConfiguration.loadConfig( - BehaviorChangeConfiguration.STORAGE_CONFIGURATION_MAX_LOCATIONS); + callContext + .getConfigurationStore() + .getConfiguration( + callContext, BehaviorChangeConfiguration.STORAGE_CONFIGURATION_MAX_LOCATIONS); if (maxAllowedLocations != -1 && allowedLocations.size() > maxAllowedLocations) { throw new IllegalArgumentException( String.format( diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java index 2170550dde..32b1955846 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java @@ -1607,7 +1607,7 @@ private void revokeGrantRecord( try { EnumMap creds = storageIntegration.getSubscopedCreds( - callCtx.getDiagServices(), + callCtx, storageConfigurationInfo, allowListOperation, allowedReadLocations, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java index 85288f0dd5..cbae446c25 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java @@ -178,21 +178,23 @@ public synchronized Supplier getOrCreateSessionSupplie @Override public synchronized StorageCredentialCache getOrCreateStorageCredentialCache( - RealmContext realmContext) { + RealmContext realmContext, PolarisCallContext polarisCallContext) { if (!storageCredentialCacheMap.containsKey(realmContext.getRealmIdentifier())) { storageCredentialCacheMap.put( - realmContext.getRealmIdentifier(), new StorageCredentialCache()); + realmContext.getRealmIdentifier(), new StorageCredentialCache(polarisCallContext)); } return storageCredentialCacheMap.get(realmContext.getRealmIdentifier()); } @Override - public synchronized EntityCache getOrCreateEntityCache(RealmContext realmContext) { + public synchronized EntityCache getOrCreateEntityCache( + RealmContext realmContext, PolarisCallContext polarisCallContext) { if (!entityCacheMap.containsKey(realmContext.getRealmIdentifier())) { PolarisMetaStoreManager metaStoreManager = getOrCreateMetaStoreManager(realmContext); entityCacheMap.put( - realmContext.getRealmIdentifier(), new InMemoryEntityCache(metaStoreManager)); + realmContext.getRealmIdentifier(), + new InMemoryEntityCache(metaStoreManager, polarisCallContext)); } return entityCacheMap.get(realmContext.getRealmIdentifier()); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java index cb2523891f..a399611363 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java @@ -20,6 +20,7 @@ import java.util.Map; import java.util.function.Supplier; +import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; import org.apache.polaris.core.persistence.cache.EntityCache; @@ -34,9 +35,11 @@ public interface MetaStoreManagerFactory { Supplier getOrCreateSessionSupplier(RealmContext realmContext); - StorageCredentialCache getOrCreateStorageCredentialCache(RealmContext realmContext); + StorageCredentialCache getOrCreateStorageCredentialCache( + RealmContext realmContext, PolarisCallContext polarisCallContext); - EntityCache getOrCreateEntityCache(RealmContext realmContext); + EntityCache getOrCreateEntityCache( + RealmContext realmContext, PolarisCallContext polarisCallContext); Map bootstrapRealms( Iterable realms, RootCredentialsSet rootCredentialsSet); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCache.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCache.java index 4599f4aed1..9e71035110 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCache.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCache.java @@ -30,7 +30,6 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.config.BehaviorChangeConfiguration; import org.apache.polaris.core.config.FeatureConfiguration; -import org.apache.polaris.core.config.PolarisConfiguration; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisGrantRecord; @@ -58,7 +57,9 @@ public class InMemoryEntityCache implements EntityCache { * * @param polarisMetaStoreManager the meta store manager implementation */ - public InMemoryEntityCache(@Nonnull PolarisMetaStoreManager polarisMetaStoreManager) { + public InMemoryEntityCache( + @Nonnull PolarisMetaStoreManager polarisMetaStoreManager, + @Nonnull PolarisCallContext polarisCallContext) { // by name cache this.byName = new ConcurrentHashMap<>(); @@ -76,7 +77,9 @@ public InMemoryEntityCache(@Nonnull PolarisMetaStoreManager polarisMetaStoreMana }; long weigherTarget = - PolarisConfiguration.loadConfig(FeatureConfiguration.ENTITY_CACHE_WEIGHER_TARGET); + polarisCallContext + .getConfigurationStore() + .getConfiguration(polarisCallContext, FeatureConfiguration.ENTITY_CACHE_WEIGHER_TARGET); Caffeine byIdBuilder = Caffeine.newBuilder() .maximumWeight(weigherTarget) @@ -84,7 +87,12 @@ public InMemoryEntityCache(@Nonnull PolarisMetaStoreManager polarisMetaStoreMana .expireAfterAccess(1, TimeUnit.HOURS) // Expire entries after 1 hour of no access .removalListener(removalListener); // Set the removal listener - if (PolarisConfiguration.loadConfig(BehaviorChangeConfiguration.ENTITY_CACHE_SOFT_VALUES)) { + boolean useSoftValues = + polarisCallContext + .getConfigurationStore() + .getConfiguration( + polarisCallContext, BehaviorChangeConfiguration.ENTITY_CACHE_SOFT_VALUES); + if (useSoftValues) { byIdBuilder.softValues(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java index 9f508eb4e1..685f2d7b15 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java @@ -2037,7 +2037,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( try { EnumMap creds = storageIntegration.getSubscopedCreds( - callCtx.getDiagServices(), + callCtx, storageConfigurationInfo, allowListOperation, allowedReadLocations, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageIntegration.java index eec9a42094..582290e997 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageIntegration.java @@ -23,7 +23,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.PolarisCallContext; /** * Abstract of Polaris Storage Integration. It holds the reference to an object that having the @@ -55,7 +55,7 @@ public String getStorageIdentifierOrId() { * @return An enum map including the scoped credentials */ public abstract EnumMap getSubscopedCreds( - @Nonnull PolarisDiagnostics diagnostics, + @Nonnull PolarisCallContext callContext, @Nonnull T storageConfig, boolean allowListOperation, @Nonnull Set allowedReadLocations, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java index 9b1c64900b..07502765e8 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java @@ -19,7 +19,6 @@ package org.apache.polaris.core.storage.aws; import static org.apache.polaris.core.config.FeatureConfiguration.STORAGE_CREDENTIAL_DURATION_SECONDS; -import static org.apache.polaris.core.config.PolarisConfiguration.loadConfig; import jakarta.annotation.Nonnull; import java.net.URI; @@ -29,7 +28,7 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Stream; -import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.storage.InMemoryStorageIntegration; import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.StorageUtil; @@ -55,11 +54,15 @@ public AwsCredentialsStorageIntegration(StsClient stsClient) { /** {@inheritDoc} */ @Override public EnumMap getSubscopedCreds( - @Nonnull PolarisDiagnostics diagnostics, + @Nonnull PolarisCallContext callContext, @Nonnull AwsStorageConfigurationInfo storageConfig, boolean allowListOperation, @Nonnull Set allowedReadLocations, @Nonnull Set allowedWriteLocations) { + int durationSeconds = + callContext + .getConfigurationStore() + .getConfiguration(callContext, STORAGE_CREDENTIAL_DURATION_SECONDS); AssumeRoleResponse response = stsClient.assumeRole( AssumeRoleRequest.builder() @@ -73,7 +76,7 @@ public EnumMap getSubscopedCreds( allowedReadLocations, allowedWriteLocations) .toJson()) - .durationSeconds(loadConfig(STORAGE_CREDENTIAL_DURATION_SECONDS)) + .durationSeconds(durationSeconds) .build()); EnumMap credentialMap = new EnumMap<>(PolarisCredentialProperty.class); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java index 62e4fc4dc1..041c40e92e 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java @@ -18,6 +18,8 @@ */ package org.apache.polaris.core.storage.azure; +import static org.apache.polaris.core.config.FeatureConfiguration.STORAGE_CREDENTIAL_DURATION_SECONDS; + import com.azure.core.credential.AccessToken; import com.azure.core.credential.TokenRequestContext; import com.azure.identity.DefaultAzureCredential; @@ -45,8 +47,7 @@ import java.util.HashSet; import java.util.Objects; import java.util.Set; -import org.apache.polaris.core.PolarisDiagnostics; -import org.apache.polaris.core.config.FeatureConfiguration; +import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.storage.InMemoryStorageIntegration; import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.slf4j.Logger; @@ -71,7 +72,7 @@ public AzureCredentialsStorageIntegration() { @Override public EnumMap getSubscopedCreds( - @Nonnull PolarisDiagnostics diagnostics, + @Nonnull PolarisCallContext callContext, @Nonnull AzureStorageConfigurationInfo storageConfig, boolean allowListOperation, @Nonnull Set allowedReadLocations, @@ -126,7 +127,9 @@ public EnumMap getSubscopedCreds( // clock skew between the client and server, OffsetDateTime startTime = start.truncatedTo(ChronoUnit.SECONDS).atOffset(ZoneOffset.UTC); int intendedDurationSeconds = - FeatureConfiguration.loadConfig(FeatureConfiguration.STORAGE_CREDENTIAL_DURATION_SECONDS); + callContext + .getConfigurationStore() + .getConfiguration(callContext, STORAGE_CREDENTIAL_DURATION_SECONDS); OffsetDateTime intendedEndTime = start.plusSeconds(intendedDurationSeconds).atOffset(ZoneOffset.UTC); OffsetDateTime maxAllowedEndTime = diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCache.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCache.java index 3fe5931950..f44c525e94 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCache.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCache.java @@ -31,7 +31,6 @@ import org.apache.iceberg.exceptions.UnprocessableEntityException; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.config.FeatureConfiguration; -import org.apache.polaris.core.config.PolarisConfiguration; import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.persistence.dao.entity.ScopedCredentialsResult; @@ -47,8 +46,11 @@ public class StorageCredentialCache { private static final long CACHE_MAX_NUMBER_OF_ENTRIES = 10_000L; private final LoadingCache cache; + private final long maxCacheDurationMs; + /** Initialize the creds cache */ - public StorageCredentialCache() { + public StorageCredentialCache(PolarisCallContext polarisCallContext) { + this.maxCacheDurationMs = maxCacheDurationMs(polarisCallContext); cache = Caffeine.newBuilder() .maximumSize(CACHE_MAX_NUMBER_OF_ENTRIES) @@ -60,7 +62,7 @@ public StorageCredentialCache() { 0, Math.min( (entry.getExpirationTime() - System.currentTimeMillis()) / 2, - maxCacheDurationMs())); + this.maxCacheDurationMs)); return Duration.ofMillis(expireAfterMillis); })) .build( @@ -71,12 +73,17 @@ public StorageCredentialCache() { } /** How long credentials should remain in the cache. */ - private static long maxCacheDurationMs() { - var cacheDurationSeconds = - PolarisConfiguration.loadConfig( - FeatureConfiguration.STORAGE_CREDENTIAL_CACHE_DURATION_SECONDS); - var credentialDurationSeconds = - PolarisConfiguration.loadConfig(FeatureConfiguration.STORAGE_CREDENTIAL_DURATION_SECONDS); + private static long maxCacheDurationMs(PolarisCallContext polarisCallContext) { + int cacheDurationSeconds = + polarisCallContext + .getConfigurationStore() + .getConfiguration( + polarisCallContext, FeatureConfiguration.STORAGE_CREDENTIAL_CACHE_DURATION_SECONDS); + int credentialDurationSeconds = + polarisCallContext + .getConfigurationStore() + .getConfiguration( + polarisCallContext, FeatureConfiguration.STORAGE_CREDENTIAL_DURATION_SECONDS); if (cacheDurationSeconds >= credentialDurationSeconds) { throw new IllegalArgumentException( String.format( diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java index 7d886e32c6..9684fac0b3 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java @@ -38,7 +38,7 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Stream; -import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.storage.InMemoryStorageIntegration; import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisStorageIntegration; @@ -70,7 +70,7 @@ public GcpCredentialsStorageIntegration( @Override public EnumMap getSubscopedCreds( - @Nonnull PolarisDiagnostics diagnostics, + @Nonnull PolarisCallContext callContext, @Nonnull GcpStorageConfigurationInfo storageConfig, boolean allowListOperation, @Nonnull Set allowedReadLocations, diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCacheTest.java index 72b75ad05b..d122846aaf 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCacheTest.java @@ -103,7 +103,7 @@ public InMemoryEntityCacheTest() { * @return new cache for the entity store */ InMemoryEntityCache allocateNewCache() { - return new InMemoryEntityCache(this.metaStoreManager); + return new InMemoryEntityCache(this.metaStoreManager, callCtx); } @Test diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java index 0f1f4f151b..5a2c697e21 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java @@ -27,7 +27,6 @@ import java.util.Set; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; -import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; @@ -208,7 +207,7 @@ public MockInMemoryStorageIntegration() { @Override public EnumMap getSubscopedCreds( - @Nonnull PolarisDiagnostics diagnostics, + @Nonnull PolarisCallContext callContext, @Nonnull PolarisStorageConfigurationInfo storageConfig, boolean allowListOperation, @Nonnull Set allowedReadLocations, diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java index 618ddddb56..33fca1c2ca 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java @@ -71,12 +71,12 @@ public StorageCredentialCacheTest() { new TreeMapTransactionalPersistenceImpl(store, Mockito.mock(), RANDOM_SECRETS); callCtx = new PolarisCallContext(metaStore, diagServices); metaStoreManager = Mockito.mock(PolarisMetaStoreManager.class); - storageCredentialCache = new StorageCredentialCache(); + storageCredentialCache = new StorageCredentialCache(callCtx); } @Test public void testBadResult() { - storageCredentialCache = new StorageCredentialCache(); + storageCredentialCache = new StorageCredentialCache(callCtx); ScopedCredentialsResult badResult = new ScopedCredentialsResult( BaseResult.ReturnStatus.SUBSCOPE_CREDS_ERROR, "extra_error_info"); @@ -109,7 +109,7 @@ public void testBadResult() { @Test public void testCacheHit() { - storageCredentialCache = new StorageCredentialCache(); + storageCredentialCache = new StorageCredentialCache(callCtx); List mockedScopedCreds = getFakeScopedCreds(3, /* expireSoon= */ false); Mockito.when( @@ -152,7 +152,7 @@ public void testCacheHit() { @RepeatedTest(10) public void testCacheEvict() throws InterruptedException { - storageCredentialCache = new StorageCredentialCache(); + storageCredentialCache = new StorageCredentialCache(callCtx); List mockedScopedCreds = getFakeScopedCreds(3, /* expireSoon= */ true); Mockito.when( @@ -210,7 +210,7 @@ public void testCacheEvict() throws InterruptedException { @Test public void testCacheGenerateNewEntries() { - storageCredentialCache = new StorageCredentialCache(); + storageCredentialCache = new StorageCredentialCache(callCtx); List mockedScopedCreds = getFakeScopedCreds(3, /* expireSoon= */ false); Mockito.when( @@ -297,7 +297,7 @@ public void testCacheGenerateNewEntries() { @Test public void testCacheNotAffectedBy() { - storageCredentialCache = new StorageCredentialCache(); + storageCredentialCache = new StorageCredentialCache(callCtx); List mockedScopedCreds = getFakeScopedCreds(3, /* expireSoon= */ false); @@ -442,7 +442,7 @@ private static List getPolarisEntities() { @Test public void testAzureCredentialFormatting() { - storageCredentialCache = new StorageCredentialCache(); + storageCredentialCache = new StorageCredentialCache(callCtx); List mockedScopedCreds = List.of( new ScopedCredentialsResult( diff --git a/polaris-core/src/test/java/org/apache/polaris/service/storage/aws/AwsCredentialsStorageIntegrationTest.java b/polaris-core/src/test/java/org/apache/polaris/service/storage/aws/AwsCredentialsStorageIntegrationTest.java index 9a68867539..78bfc87553 100644 --- a/polaris-core/src/test/java/org/apache/polaris/service/storage/aws/AwsCredentialsStorageIntegrationTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/service/storage/aws/AwsCredentialsStorageIntegrationTest.java @@ -22,20 +22,25 @@ import jakarta.annotation.Nonnull; import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; import java.util.EnumMap; import java.util.List; import java.util.Set; -import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.aws.AwsCredentialsStorageIntegration; import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; import org.assertj.core.api.Assertions; import org.assertj.core.api.InstanceOfAssertFactories; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mockito; +import org.threeten.extra.MutableClock; import software.amazon.awssdk.policybuilder.iam.IamAction; import software.amazon.awssdk.policybuilder.iam.IamCondition; import software.amazon.awssdk.policybuilder.iam.IamConditionOperator; @@ -64,6 +69,18 @@ class AwsCredentialsStorageIntegrationTest { .build(); public static final String AWS_PARTITION = "aws"; + public PolarisCallContext callContext; + + @BeforeEach + public void setup() { + callContext = + new PolarisCallContext( + Mockito.mock(), + Mockito.mock(), + new PolarisConfigurationStore() {}, + MutableClock.of(Instant.now(), ZoneOffset.UTC).withZone(ZoneId.systemDefault())); + } + @Test public void testGetSubscopedCreds() { StsClient stsClient = Mockito.mock(StsClient.class); @@ -83,7 +100,7 @@ public void testGetSubscopedCreds() { EnumMap credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + callContext, new AwsStorageConfigurationInfo( PolarisStorageConfigurationInfo.StorageType.S3, List.of(warehouseDir), @@ -231,7 +248,7 @@ public void testGetSubscopedCredsInlinePolicy(String awsPartition) { () -> new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + callContext, new AwsStorageConfigurationInfo( storageType, List.of(s3Path(bucket, warehouseKeyPrefix)), @@ -248,7 +265,7 @@ public void testGetSubscopedCredsInlinePolicy(String awsPartition) { EnumMap credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + callContext, new AwsStorageConfigurationInfo( storageType, List.of(s3Path(bucket, warehouseKeyPrefix)), @@ -349,7 +366,7 @@ public void testGetSubscopedCredsInlinePolicyWithoutList() { EnumMap credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + callContext, new AwsStorageConfigurationInfo( PolarisStorageConfigurationInfo.StorageType.S3, List.of(s3Path(bucket, warehouseKeyPrefix)), @@ -444,7 +461,7 @@ public void testGetSubscopedCredsInlinePolicyWithoutWrites() { EnumMap credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + callContext, new AwsStorageConfigurationInfo( storageType, List.of(s3Path(bucket, warehouseKeyPrefix)), @@ -509,7 +526,7 @@ public void testGetSubscopedCredsInlinePolicyWithEmptyReadAndWrite() { EnumMap credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + callContext, new AwsStorageConfigurationInfo( PolarisStorageConfigurationInfo.StorageType.S3, List.of(s3Path(bucket, warehouseKeyPrefix)), @@ -549,7 +566,7 @@ public void testClientRegion(String awsPartition) { () -> new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + callContext, new AwsStorageConfigurationInfo( PolarisStorageConfigurationInfo.StorageType.S3, List.of(s3Path(bucket, warehouseKeyPrefix)), @@ -566,7 +583,7 @@ public void testClientRegion(String awsPartition) { EnumMap credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + callContext, new AwsStorageConfigurationInfo( PolarisStorageConfigurationInfo.StorageType.S3, List.of(s3Path(bucket, warehouseKeyPrefix)), @@ -604,7 +621,7 @@ public void testNoClientRegion(String awsPartition) { EnumMap credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + callContext, new AwsStorageConfigurationInfo( PolarisStorageConfigurationInfo.StorageType.S3, List.of(s3Path(bucket, warehouseKeyPrefix)), @@ -624,7 +641,7 @@ public void testNoClientRegion(String awsPartition) { () -> new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + callContext, new AwsStorageConfigurationInfo( PolarisStorageConfigurationInfo.StorageType.S3, List.of(s3Path(bucket, warehouseKeyPrefix)), diff --git a/polaris-core/src/test/java/org/apache/polaris/service/storage/azure/AzureCredentialStorageIntegrationTest.java b/polaris-core/src/test/java/org/apache/polaris/service/storage/azure/AzureCredentialStorageIntegrationTest.java index fd7ac9a880..1a66151256 100644 --- a/polaris-core/src/test/java/org/apache/polaris/service/storage/azure/AzureCredentialStorageIntegrationTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/service/storage/azure/AzureCredentialStorageIntegrationTest.java @@ -40,6 +40,9 @@ import java.lang.annotation.Target; import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Arrays; import java.util.EnumMap; @@ -47,7 +50,8 @@ import java.util.List; import java.util.Map; import java.util.stream.Stream; -import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.azure.AzureCredentialsStorageIntegration; import org.apache.polaris.core.storage.azure.AzureStorageConfigurationInfo; @@ -60,6 +64,8 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.ArgumentsProvider; import org.junit.jupiter.params.provider.ArgumentsSource; +import org.mockito.Mockito; +import org.threeten.extra.MutableClock; public class AzureCredentialStorageIntegrationTest { @@ -349,7 +355,11 @@ private Map subscopedCredsForOperations( new AzureCredentialsStorageIntegration(); EnumMap credsMap = azureCredsIntegration.getSubscopedCreds( - new PolarisDefaultDiagServiceImpl(), + new PolarisCallContext( + Mockito.mock(), + Mockito.mock(), + new PolarisConfigurationStore() {}, + MutableClock.of(Instant.now(), ZoneOffset.UTC).withZone(ZoneId.systemDefault())), azureConfig, allowListAction, new HashSet<>(allowedReadLoc), diff --git a/polaris-core/src/test/java/org/apache/polaris/service/storage/gcp/GcpCredentialsStorageIntegrationTest.java b/polaris-core/src/test/java/org/apache/polaris/service/storage/gcp/GcpCredentialsStorageIntegrationTest.java index 35d05c2513..498609da4f 100644 --- a/polaris-core/src/test/java/org/apache/polaris/service/storage/gcp/GcpCredentialsStorageIntegrationTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/service/storage/gcp/GcpCredentialsStorageIntegrationTest.java @@ -39,6 +39,8 @@ import com.google.cloud.storage.StorageOptions; import java.io.IOException; import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Arrays; @@ -48,7 +50,8 @@ import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.gcp.GcpCredentialsStorageIntegration; import org.apache.polaris.core.storage.gcp.GcpStorageConfigurationInfo; @@ -58,6 +61,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.Mockito; +import org.threeten.extra.MutableClock; class GcpCredentialsStorageIntegrationTest { @@ -172,7 +177,11 @@ private Map subscopedCredsForOperations( ServiceOptions.getFromServiceLoader(HttpTransportFactory.class, NetHttpTransport::new)); EnumMap credsMap = gcpCredsIntegration.getSubscopedCreds( - new PolarisDefaultDiagServiceImpl(), + new PolarisCallContext( + Mockito.mock(), + Mockito.mock(), + new PolarisConfigurationStore() {}, + MutableClock.of(Instant.now(), ZoneOffset.UTC).withZone(ZoneId.systemDefault())), gcpConfig, allowListAction, new HashSet<>(allowedReadLoc), diff --git a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BaseResolverTest.java b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BaseResolverTest.java index 728187a38d..5f08391cee 100644 --- a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BaseResolverTest.java +++ b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BaseResolverTest.java @@ -468,7 +468,7 @@ private Resolver allocateResolver( // create a new cache if needs be if (cache == null) { - this.cache = new InMemoryEntityCache(metaStoreManager()); + this.cache = new InMemoryEntityCache(metaStoreManager(), callCtx()); } boolean allRoles = principalRolesScope == null; Optional> roleEntities = diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java index 17672ce2bc..d15a0db159 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java @@ -89,8 +89,8 @@ public Clock clock() { @Produces @ApplicationScoped - public StorageCredentialCache storageCredentialCache() { - return new StorageCredentialCache(); + public StorageCredentialCache storageCredentialCache(PolarisCallContext polarisCallContext) { + return new StorageCredentialCache(polarisCallContext); } @Produces @@ -264,8 +264,10 @@ public BasePersistence polarisMetaStoreSession( @Produces @RequestScoped public PolarisEntityManager polarisEntityManager( - RealmContext realmContext, RealmEntityManagerFactory factory) { - return factory.getOrCreateEntityManager(realmContext); + RealmContext realmContext, + RealmEntityManagerFactory factory, + PolarisCallContext polarisCallContext) { + return factory.getOrCreateEntityManager(realmContext, polarisCallContext); } @Produces diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java index 6f3498d319..2112a055d1 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java @@ -236,7 +236,8 @@ public void before(TestInfo testInfo) { } }, clock); - this.entityManager = realmEntityManagerFactory.getOrCreateEntityManager(realmContext); + this.entityManager = + realmEntityManagerFactory.getOrCreateEntityManager(realmContext, polarisContext); callContext = CallContext.of(realmContext, polarisContext); CallContext.setCurrentContext(callContext); @@ -277,7 +278,8 @@ public void before(TestInfo testInfo) { .setName(CATALOG_NAME) .setCatalogType("INTERNAL") .setDefaultBaseLocation(storageLocation) - .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .setStorageConfigurationInfo( + storageConfigModel, storageLocation, polarisContext) .build() .asCatalog())); diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/CatalogNoEntityCacheTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/CatalogNoEntityCacheTest.java index 895b6ab81e..4af23f3df1 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/CatalogNoEntityCacheTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/CatalogNoEntityCacheTest.java @@ -21,6 +21,7 @@ import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; import jakarta.annotation.Nullable; +import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.cache.InMemoryEntityCache; @@ -30,7 +31,8 @@ public class CatalogNoEntityCacheTest extends IcebergCatalogTest { @Nullable @Override - protected InMemoryEntityCache createEntityCache(PolarisMetaStoreManager metaStoreManager) { + protected InMemoryEntityCache createEntityCache( + PolarisMetaStoreManager metaStoreManager, PolarisCallContext polarisCallContext) { return null; } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java index 96b1010b43..46dd7015b7 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java @@ -172,8 +172,8 @@ public void before(TestInfo testInfo) { entityManager = new PolarisEntityManager( metaStoreManager, - new StorageCredentialCache(), - new InMemoryEntityCache(metaStoreManager)); + new StorageCredentialCache(polarisContext), + new InMemoryEntityCache(metaStoreManager, polarisContext)); callContext = CallContext.of(realmContext, polarisContext); @@ -224,7 +224,8 @@ public void before(TestInfo testInfo) { .addProperty( FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .setStorageConfigurationInfo( + storageConfigModel, storageLocation, polarisContext) .build() .asCatalog())); @@ -290,13 +291,15 @@ public Supplier getOrCreateSessionSupplier( } @Override - public StorageCredentialCache getOrCreateStorageCredentialCache(RealmContext realmContext) { - return new StorageCredentialCache(); + public StorageCredentialCache getOrCreateStorageCredentialCache( + RealmContext realmContext, PolarisCallContext polarisCallContext) { + return new StorageCredentialCache(polarisCallContext); } @Override - public InMemoryEntityCache getOrCreateEntityCache(RealmContext realmContext) { - return new InMemoryEntityCache(metaStoreManager); + public InMemoryEntityCache getOrCreateEntityCache( + RealmContext realmContext, PolarisCallContext polaraisCallContext) { + return new InMemoryEntityCache(metaStoreManager, polaraisCallContext); } @Override diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java index 11a938f316..8a16b24290 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java @@ -50,6 +50,7 @@ import org.apache.iceberg.rest.requests.UpdateTableRequest; import org.apache.iceberg.view.ImmutableSQLViewRepresentation; import org.apache.iceberg.view.ImmutableViewVersion; +import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.admin.model.CreateCatalogRequest; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials; @@ -1732,7 +1733,8 @@ public void testSendNotificationSufficientPrivileges() { new CatalogEntity.Builder() .setName(externalCatalog) .setDefaultBaseLocation(storageLocation) - .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .setStorageConfigurationInfo( + storageConfigModel, storageLocation, callContext.getPolarisCallContext()) .setCatalogType("EXTERNAL") .build() .asCatalog())); @@ -1800,7 +1802,8 @@ public void testSendNotificationSufficientPrivileges() { new PolarisCallContextCatalogFactory( new RealmEntityManagerFactory(null) { @Override - public PolarisEntityManager getOrCreateEntityManager(RealmContext realmContext) { + public PolarisEntityManager getOrCreateEntityManager( + RealmContext realmContext, PolarisCallContext polarisCallContext) { return entityManager; } }, diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java index 5a83330a5b..11a3059f96 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java @@ -220,7 +220,7 @@ public static void setUpMocks() { @Nullable protected abstract InMemoryEntityCache createEntityCache( - PolarisMetaStoreManager metaStoreManager); + PolarisMetaStoreManager metaStoreManager, PolarisCallContext polarisCallContext); @BeforeEach @SuppressWarnings("unchecked") @@ -240,7 +240,9 @@ public void before(TestInfo testInfo) { Clock.systemDefaultZone()); entityManager = new PolarisEntityManager( - metaStoreManager, new StorageCredentialCache(), createEntityCache(metaStoreManager)); + metaStoreManager, + new StorageCredentialCache(polarisContext), + createEntityCache(metaStoreManager, polarisContext)); callContext = CallContext.of(realmContext, polarisContext); @@ -292,7 +294,8 @@ public void before(TestInfo testInfo) { .addProperty( FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .setStorageConfigurationInfo( + storageConfigModel, storageLocation, polarisContext) .build() .asCatalog())); @@ -397,13 +400,15 @@ public Supplier getOrCreateSessionSupplier( } @Override - public StorageCredentialCache getOrCreateStorageCredentialCache(RealmContext realmContext) { - return new StorageCredentialCache(); + public StorageCredentialCache getOrCreateStorageCredentialCache( + RealmContext realmContext, PolarisCallContext polarisCallContext) { + return new StorageCredentialCache(polarisCallContext); } @Override - public InMemoryEntityCache getOrCreateEntityCache(RealmContext realmContext) { - return new InMemoryEntityCache(metaStoreManager); + public InMemoryEntityCache getOrCreateEntityCache( + RealmContext realmContext, PolarisCallContext polarisCallContext) { + return new InMemoryEntityCache(metaStoreManager, polarisCallContext); } @Override @@ -1573,7 +1578,8 @@ public void testDropTableWithPurgeDisabled() { .addProperty( FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") .addProperty(FeatureConfiguration.DROP_WITH_PURGE_ENABLED.catalogConfig(), "false") - .setStorageConfigurationInfo(noPurgeStorageConfigModel, storageLocation) + .setStorageConfigurationInfo( + noPurgeStorageConfigModel, storageLocation, polarisContext) .build() .asCatalog())); PolarisPassthroughResolutionView passthroughView = diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java index 25089d5c4f..9845af7e68 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java @@ -159,8 +159,8 @@ public void before(TestInfo testInfo) { PolarisEntityManager entityManager = new PolarisEntityManager( metaStoreManager, - new StorageCredentialCache(), - new InMemoryEntityCache(metaStoreManager)); + new StorageCredentialCache(polarisContext), + new InMemoryEntityCache(metaStoreManager, polarisContext)); CallContext callContext = CallContext.of(realmContext, polarisContext); CallContext.setCurrentContext(callContext); @@ -202,7 +202,8 @@ public void before(TestInfo testInfo) { .setStorageConfigurationInfo( new FileStorageConfigInfo( StorageConfigInfo.StorageTypeEnum.FILE, List.of("file://", "/", "*")), - "file://tmp") + "file://tmp", + polarisContext) .build() .asCatalog())); diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogWithEntityCacheTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogWithEntityCacheTest.java index 079b02befc..d3a9ff3d61 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogWithEntityCacheTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogWithEntityCacheTest.java @@ -21,6 +21,7 @@ import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; import jakarta.annotation.Nullable; +import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.cache.InMemoryEntityCache; @@ -30,7 +31,8 @@ public class PolarisCatalogWithEntityCacheTest extends IcebergCatalogTest { @Nullable @Override - protected InMemoryEntityCache createEntityCache(PolarisMetaStoreManager metaStoreManager) { - return new InMemoryEntityCache(metaStoreManager); + protected InMemoryEntityCache createEntityCache( + PolarisMetaStoreManager metaStoreManager, PolarisCallContext polarisCallContext) { + return new InMemoryEntityCache(metaStoreManager, polarisCallContext); } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java index a5bf98ba99..e6476a6472 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java @@ -196,8 +196,8 @@ public void before(TestInfo testInfo) { entityManager = new PolarisEntityManager( metaStoreManager, - new StorageCredentialCache(), - new InMemoryEntityCache(metaStoreManager)); + new StorageCredentialCache(polarisContext), + new InMemoryEntityCache(metaStoreManager, polarisContext)); callContext = CallContext.of(realmContext, polarisContext); @@ -248,7 +248,8 @@ public void before(TestInfo testInfo) { .addProperty( FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .setStorageConfigurationInfo( + storageConfigModel, storageLocation, polarisContext) .build() .asCatalog())); @@ -313,13 +314,15 @@ public Supplier getOrCreateSessionSupplier( } @Override - public StorageCredentialCache getOrCreateStorageCredentialCache(RealmContext realmContext) { - return new StorageCredentialCache(); + public StorageCredentialCache getOrCreateStorageCredentialCache( + RealmContext realmContext, PolarisCallContext polarisCallContext) { + return new StorageCredentialCache(polarisCallContext); } @Override - public InMemoryEntityCache getOrCreateEntityCache(RealmContext realmContext) { - return new InMemoryEntityCache(metaStoreManager); + public InMemoryEntityCache getOrCreateEntityCache( + RealmContext realmContext, PolarisCallContext polarisCallContext) { + return new InMemoryEntityCache(metaStoreManager, polarisCallContext); } @Override diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/entity/CatalogEntityTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/entity/CatalogEntityTest.java index 8d8caabfd6..756ce68573 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/entity/CatalogEntityTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/entity/CatalogEntityTest.java @@ -33,17 +33,19 @@ import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; public class CatalogEntityTest { - @BeforeAll - public static void setup() { + private PolarisCallContext polarisCallContext; + + @BeforeEach + public void setup() { MetaStoreManagerFactory metaStoreManagerFactory = new InMemoryPolarisMetaStoreManagerFactory(); - PolarisCallContext polarisCallContext = + this.polarisCallContext = new PolarisCallContext( metaStoreManagerFactory.getOrCreateSessionSupplier(() -> "realm").get(), new PolarisDefaultDiagServiceImpl()); @@ -70,7 +72,7 @@ public void testInvalidAllowedLocationPrefix() { .setProperties(prop) .setStorageConfigInfo(awsStorageConfigModel) .build(); - Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(awsCatalog)) + Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(polarisCallContext, awsCatalog)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining( "Location prefix not allowed: 'unsupportPrefix://mybucket/path', expected prefixes"); @@ -91,7 +93,7 @@ public void testInvalidAllowedLocationPrefix() { new CatalogProperties("abfs://container@storageaccount.blob.windows.net/path")) .setStorageConfigInfo(azureStorageConfigModel) .build(); - Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(azureCatalog)) + Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(polarisCallContext, azureCatalog)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("Invalid azure location uri unsupportPrefix://mybucket/path"); @@ -108,7 +110,7 @@ public void testInvalidAllowedLocationPrefix() { .setProperties(new CatalogProperties("gs://externally-owned-bucket")) .setStorageConfigInfo(gcpStorageConfigModel) .build(); - Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(gcpCatalog)) + Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(polarisCallContext, gcpCatalog)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining( "Location prefix not allowed: 'unsupportPrefix://mybucket/path', expected prefixes"); @@ -140,7 +142,7 @@ public void testExceedMaxAllowedLocations() { .setProperties(prop) .setStorageConfigInfo(awsStorageConfigModel) .build(); - Assertions.assertThatCode(() -> CatalogEntity.fromCatalog(awsCatalog)) + Assertions.assertThatCode(() -> CatalogEntity.fromCatalog(polarisCallContext, awsCatalog)) .doesNotThrowAnyException(); } @@ -164,7 +166,8 @@ public void testValidAllowedLocationPrefix() { .setProperties(prop) .setStorageConfigInfo(awsStorageConfigModel) .build(); - Assertions.assertThatNoException().isThrownBy(() -> CatalogEntity.fromCatalog(awsCatalog)); + Assertions.assertThatNoException() + .isThrownBy(() -> CatalogEntity.fromCatalog(polarisCallContext, awsCatalog)); basedLocation = "abfs://container@storageaccount.blob.windows.net/path"; prop.put(CatalogEntity.DEFAULT_BASE_LOCATION_KEY, basedLocation); @@ -181,7 +184,8 @@ public void testValidAllowedLocationPrefix() { .setProperties(new CatalogProperties(basedLocation)) .setStorageConfigInfo(azureStorageConfigModel) .build(); - Assertions.assertThatNoException().isThrownBy(() -> CatalogEntity.fromCatalog(azureCatalog)); + Assertions.assertThatNoException() + .isThrownBy(() -> CatalogEntity.fromCatalog(polarisCallContext, azureCatalog)); basedLocation = "gs://externally-owned-bucket"; prop.put(CatalogEntity.DEFAULT_BASE_LOCATION_KEY, basedLocation); @@ -197,7 +201,8 @@ public void testValidAllowedLocationPrefix() { .setProperties(new CatalogProperties(basedLocation)) .setStorageConfigInfo(gcpStorageConfigModel) .build(); - Assertions.assertThatNoException().isThrownBy(() -> CatalogEntity.fromCatalog(gcpCatalog)); + Assertions.assertThatNoException() + .isThrownBy(() -> CatalogEntity.fromCatalog(polarisCallContext, gcpCatalog)); } @ParameterizedTest @@ -232,7 +237,7 @@ public void testInvalidArn(String roleArn) { expectedMessage = "Invalid role ARN format"; } ; - Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(awsCatalog)) + Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(polarisCallContext, awsCatalog)) .isInstanceOf(IllegalArgumentException.class) .hasMessage(expectedMessage); } diff --git a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java index b814d902fe..daffd1cf84 100644 --- a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java +++ b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java @@ -666,7 +666,8 @@ public PolarisEntity createCatalog(CreateCatalogRequest catalogRequest) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.CREATE_CATALOG; authorizeBasicRootOperationOrThrow(op); - CatalogEntity entity = CatalogEntity.fromCatalog(catalogRequest.getCatalog()); + CatalogEntity entity = + CatalogEntity.fromCatalog(callContext.getPolarisCallContext(), catalogRequest.getCatalog()); checkArgument(entity.getId() == -1, "Entity to be created must have no ID assigned"); @@ -843,7 +844,9 @@ private void validateUpdateCatalogDiffOrThrow( } if (updateRequest.getStorageConfigInfo() != null) { updateBuilder.setStorageConfigurationInfo( - updateRequest.getStorageConfigInfo(), defaultBaseLocation); + updateRequest.getStorageConfigInfo(), + defaultBaseLocation, + callContext.getPolarisCallContext()); } CatalogEntity updatedEntity = updateBuilder.build(); diff --git a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java index c2c53c17b2..b2acbcc71a 100644 --- a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java +++ b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java @@ -115,7 +115,8 @@ private PolarisAdminService newAdminService( } PolarisEntityManager entityManager = - entityManagerFactory.getOrCreateEntityManager(realmContext); + entityManagerFactory.getOrCreateEntityManager( + realmContext, callContext.getPolarisCallContext()); PolarisMetaStoreManager metaStoreManager = metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); UserSecretsManager userSecretsManager = diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java b/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java index 31f2f9b03a..64e2331cc9 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java @@ -79,7 +79,8 @@ public FileIO loadFileIO( @Nonnull PolarisResolvedPathWrapper resolvedEntityPath) { RealmContext realmContext = callContext.getRealmContext(); PolarisEntityManager entityManager = - realmEntityManagerFactory.getOrCreateEntityManager(realmContext); + realmEntityManagerFactory.getOrCreateEntityManager( + realmContext, callContext.getPolarisCallContext()); PolarisCredentialVendor credentialVendor = metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); diff --git a/service/common/src/main/java/org/apache/polaris/service/config/RealmEntityManagerFactory.java b/service/common/src/main/java/org/apache/polaris/service/config/RealmEntityManagerFactory.java index 03799bd401..f779ce18a3 100644 --- a/service/common/src/main/java/org/apache/polaris/service/config/RealmEntityManagerFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/config/RealmEntityManagerFactory.java @@ -22,6 +22,7 @@ import jakarta.inject.Inject; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; @@ -44,7 +45,8 @@ public RealmEntityManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory this.metaStoreManagerFactory = metaStoreManagerFactory; } - public PolarisEntityManager getOrCreateEntityManager(RealmContext context) { + public PolarisEntityManager getOrCreateEntityManager( + RealmContext context, PolarisCallContext polarisCallContext) { String realm = context.getRealmIdentifier(); LOGGER.debug("Looking up PolarisEntityManager for realm {}", realm); @@ -55,8 +57,9 @@ public PolarisEntityManager getOrCreateEntityManager(RealmContext context) { LOGGER.info("Initializing new PolarisEntityManager for realm {}", r); return new PolarisEntityManager( metaStoreManagerFactory.getOrCreateMetaStoreManager(context), - metaStoreManagerFactory.getOrCreateStorageCredentialCache(context), - metaStoreManagerFactory.getOrCreateEntityCache(context)); + metaStoreManagerFactory.getOrCreateStorageCredentialCache( + context, polarisCallContext), + metaStoreManagerFactory.getOrCreateEntityCache(context, polarisCallContext)); }); } } diff --git a/service/common/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java b/service/common/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java index 94fe197609..52266f9040 100644 --- a/service/common/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java @@ -85,7 +85,8 @@ public Catalog createCallContextCatalog( LOGGER.debug("Initializing new BasePolarisCatalog for key: {}", catalogKey); PolarisEntityManager entityManager = - entityManagerFactory.getOrCreateEntityManager(context.getRealmContext()); + entityManagerFactory.getOrCreateEntityManager( + context.getRealmContext(), context.getPolarisCallContext()); IcebergCatalog catalogInstance = new IcebergCatalog( diff --git a/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java b/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java index f61c67620f..cef3158007 100644 --- a/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java +++ b/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java @@ -30,7 +30,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Supplier; -import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; @@ -90,7 +90,7 @@ public PolarisStorageIntegrationProviderImpl( new PolarisStorageIntegration<>("file") { @Override public EnumMap getSubscopedCreds( - @Nonnull PolarisDiagnostics diagnostics, + @Nonnull PolarisCallContext callContext, @Nonnull T storageConfig, boolean allowListOperation, @Nonnull Set allowedReadLocations, diff --git a/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java b/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java index 9a070273b8..e0f684590b 100644 --- a/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java +++ b/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java @@ -233,12 +233,16 @@ IcebergCatalog createCatalog(TestServices services) { PolarisPassthroughResolutionView passthroughView = new PolarisPassthroughResolutionView( callContext, - services.entityManagerFactory().getOrCreateEntityManager(realmContext), + services + .entityManagerFactory() + .getOrCreateEntityManager(realmContext, callContext.getPolarisCallContext()), services.securityContext(), CATALOG_NAME); IcebergCatalog polarisCatalog = new IcebergCatalog( - services.entityManagerFactory().getOrCreateEntityManager(realmContext), + services + .entityManagerFactory() + .getOrCreateEntityManager(realmContext, callContext.getPolarisCallContext()), services.metaStoreManagerFactory().getOrCreateMetaStoreManager(realmContext), callContext, passthroughView, diff --git a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java index 88406b8954..b455255e88 100644 --- a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java +++ b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java @@ -140,6 +140,16 @@ public TestServices build() { BasePersistence metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); + + PolarisCallContext polarisCallContext = + new PolarisCallContext( + metaStoreSession, polarisDiagnostics, configurationStore, Mockito.mock(Clock.class)); + + PolarisEntityManager entityManager = + realmEntityManagerFactory.getOrCreateEntityManager(realmContext, polarisCallContext); + PolarisMetaStoreManager metaStoreManager = + metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); + CallContext callContext = new CallContext() { @Override @@ -149,11 +159,7 @@ public RealmContext getRealmContext() { @Override public PolarisCallContext getPolarisCallContext() { - return new PolarisCallContext( - metaStoreSession, - polarisDiagnostics, - configurationStore, - Mockito.mock(Clock.class)); + return polarisCallContext; } @Override @@ -162,10 +168,6 @@ public Map contextVariables() { } }; CallContext.setCurrentContext(callContext); - PolarisEntityManager entityManager = - realmEntityManagerFactory.getOrCreateEntityManager(realmContext); - PolarisMetaStoreManager metaStoreManager = - metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); UserSecretsManager userSecretsManager = userSecretsManagerFactory.getOrCreateUserSecretsManager(realmContext);