Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import jakarta.annotation.Nonnull;
import jakarta.annotation.Nullable;
import java.util.ArrayList;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
Expand Down Expand Up @@ -71,9 +70,9 @@
import org.apache.polaris.core.policy.PolicyEntity;
import org.apache.polaris.core.policy.PolicyMappingUtil;
import org.apache.polaris.core.policy.PolicyType;
import org.apache.polaris.core.storage.AccessConfig;
import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo;
import org.apache.polaris.core.storage.PolarisStorageIntegration;
import org.apache.polaris.core.storage.StorageAccessProperty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -1614,14 +1613,14 @@ private void revokeGrantRecord(
PolarisStorageConfigurationInfo storageConfigurationInfo =
BaseMetaStoreManager.extractStorageConfiguration(callCtx, reloadedEntity.getEntity());
try {
EnumMap<StorageAccessProperty, String> creds =
AccessConfig accessConfig =
storageIntegration.getSubscopedCreds(
callCtx,
storageConfigurationInfo,
allowListOperation,
allowedReadLocations,
allowedWriteLocations);
return new ScopedCredentialsResult(creds);
return new ScopedCredentialsResult(accessConfig);
} catch (Exception ex) {
return new ScopedCredentialsResult(
BaseResult.ReturnStatus.SUBSCOPE_CREDS_ERROR, ex.getMessage());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,15 @@
*/
package org.apache.polaris.core.persistence.dao.entity;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import jakarta.annotation.Nonnull;
import jakarta.annotation.Nullable;
import java.util.EnumMap;
import java.util.Map;
import org.apache.polaris.core.storage.StorageAccessProperty;
import org.apache.polaris.core.storage.AccessConfig;

/** Result of a getSubscopedCredsForEntity() call */
public class ScopedCredentialsResult extends BaseResult {

// null if not success. Else, set of name/value pairs for the credentials
private final EnumMap<StorageAccessProperty, String> credentials;
private final AccessConfig accessConfig;

/**
* Constructor for an error
Expand All @@ -41,32 +37,20 @@ public class ScopedCredentialsResult extends BaseResult {
public ScopedCredentialsResult(
@Nonnull ReturnStatus errorCode, @Nullable String extraInformation) {
super(errorCode, extraInformation);
this.credentials = null;
this.accessConfig = null;
}

/**
* Constructor for success
*
* @param credentials credentials
* @param accessConfig credentials
*/
public ScopedCredentialsResult(@Nonnull EnumMap<StorageAccessProperty, String> credentials) {
public ScopedCredentialsResult(AccessConfig accessConfig) {
super(ReturnStatus.SUCCESS);
this.credentials = credentials;
this.accessConfig = accessConfig;
}

@JsonCreator
private ScopedCredentialsResult(
@JsonProperty("returnStatus") @Nonnull ReturnStatus returnStatus,
@JsonProperty("extraInformation") String extraInformation,
@JsonProperty("credentials") Map<String, String> credentials) {
super(returnStatus, extraInformation);
this.credentials = new EnumMap<>(StorageAccessProperty.class);
if (credentials != null) {
credentials.forEach((k, v) -> this.credentials.put(StorageAccessProperty.valueOf(k), v));
}
}

public EnumMap<StorageAccessProperty, String> getCredentials() {
return credentials;
public AccessConfig getAccessConfig() {
return accessConfig;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import jakarta.annotation.Nonnull;
import jakarta.annotation.Nullable;
import java.util.ArrayList;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
Expand Down Expand Up @@ -76,9 +75,9 @@
import org.apache.polaris.core.policy.PolicyEntity;
import org.apache.polaris.core.policy.PolicyMappingUtil;
import org.apache.polaris.core.policy.PolicyType;
import org.apache.polaris.core.storage.AccessConfig;
import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo;
import org.apache.polaris.core.storage.PolarisStorageIntegration;
import org.apache.polaris.core.storage.StorageAccessProperty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -2062,14 +2061,14 @@ private PolarisEntityResolver resolveSecurableToRoleGrant(
PolarisStorageConfigurationInfo storageConfigurationInfo =
BaseMetaStoreManager.extractStorageConfiguration(callCtx, reloadedEntity.getEntity());
try {
EnumMap<StorageAccessProperty, String> creds =
AccessConfig accessConfig =
storageIntegration.getSubscopedCreds(
callCtx,
storageConfigurationInfo,
allowListOperation,
allowedReadLocations,
allowedWriteLocations);
return new ScopedCredentialsResult(creds);
return new ScopedCredentialsResult(accessConfig);
} catch (Exception ex) {
return new ScopedCredentialsResult(
BaseResult.ReturnStatus.SUBSCOPE_CREDS_ERROR, ex.getMessage());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,10 @@
*/
package org.apache.polaris.core.storage;

import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.time.Instant;
import java.util.Map;
import java.util.Optional;
import org.apache.polaris.immutables.PolarisImmutable;

@PolarisImmutable
Expand All @@ -27,7 +30,52 @@ public interface AccessConfig {

Map<String, String> extraProperties();

static ImmutableAccessConfig.Builder builder() {
/**
* Configuration properties that are relevant only to the Polaris Server, but not to clients.
* These properties override corresponding entries from {@link #extraProperties()}.
*/
Map<String, String> internalProperties();

Optional<Instant> expiresAt();

default String get(StorageAccessProperty key) {
if (key.isCredential()) {
return credentials().get(key.getPropertyName());
} else {
String value = internalProperties().get(key.getPropertyName());
return value != null ? value : extraProperties().get(key.getPropertyName());
}
}

static AccessConfig.Builder builder() {
return ImmutableAccessConfig.builder();
}

interface Builder {
@CanIgnoreReturnValue
Builder putCredential(String key, String value);

@CanIgnoreReturnValue
Builder putExtraProperty(String key, String value);

@CanIgnoreReturnValue
Builder putInternalProperty(String key, String value);

@CanIgnoreReturnValue
Builder expiresAt(Instant expiresAt);

default Builder put(StorageAccessProperty key, String value) {
if (key.isExpirationTimestamp()) {
expiresAt(Instant.ofEpochMilli(Long.parseLong(value)));
}

if (key.isCredential()) {
return putCredential(key.getPropertyName(), value);
} else {
return putExtraProperty(key.getPropertyName(), value);
}
}

AccessConfig build();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
package org.apache.polaris.core.storage;

import jakarta.annotation.Nonnull;
import java.util.EnumMap;
import java.util.Map;
import java.util.Set;
import org.apache.polaris.core.context.CallContext;
Expand Down Expand Up @@ -53,7 +52,7 @@ public String getStorageIdentifierOrId() {
* @param allowedWriteLocations a set of allowed to write locations
* @return An enum map including the scoped credentials
*/
public abstract EnumMap<StorageAccessProperty, String> getSubscopedCreds(
public abstract AccessConfig getSubscopedCreds(
@Nonnull CallContext callContext,
@Nonnull T storageConfig,
boolean allowListOperation,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,9 @@ public enum StorageAccessProperty {
AWS_SESSION_TOKEN_EXPIRES_AT_MS(
String.class,
"s3.session-token-expires-at-ms",
"the time the aws session token expires, in milliseconds"),
"the time the aws session token expires, in milliseconds",
true,
true),
AWS_ENDPOINT(String.class, "s3.endpoint", "the S3 endpoint to use for requests", false),
AWS_PATH_STYLE_ACCESS(
Boolean.class, "s3.path-style-access", "whether to use S3 path style access", false),
Expand All @@ -42,23 +44,26 @@ public enum StorageAccessProperty {
GCS_ACCESS_TOKEN_EXPIRES_AT(
String.class,
"gcs.oauth2.token-expires-at",
"the time the gcs access token expires, in milliseconds"),
"the time the gcs access token expires, in milliseconds",
true,
true),

// Currently not using ACCESS TOKEN as the ResolvingFileIO is using ADLSFileIO for azure case and
// it expects for SAS
AZURE_ACCESS_TOKEN(String.class, "", "the azure scoped access token"),
AZURE_SAS_TOKEN(String.class, "adls.sas-token.", "an azure shared access signature token"),
AZURE_ACCOUNT_HOST(
String.class,
"the azure storage account host",
"the azure account name + endpoint that will append to the ADLS_SAS_TOKEN_PREFIX"),
EXPIRATION_TIME(
Long.class, "expiration-time", "the expiration time for the access token, in milliseconds");
Long.class,
"expiration-time",
"the expiration time for the access token, in milliseconds",
true,
true);

private final Class valueType;
private final String propertyName;
private final String description;
private final boolean isCredential;
private final boolean isExpirationTimestamp;

/*
s3.access-key-id`: id for for credentials that provide access to the data in S3
Expand All @@ -71,10 +76,20 @@ public enum StorageAccessProperty {

StorageAccessProperty(
Class valueType, String propertyName, String description, boolean isCredential) {
this(valueType, propertyName, description, isCredential, false);
}

StorageAccessProperty(
Class valueType,
String propertyName,
String description,
boolean isCredential,
boolean isExpirationTimestamp) {
this.valueType = valueType;
this.propertyName = propertyName;
this.description = description;
this.isCredential = isCredential;
this.isExpirationTimestamp = isExpirationTimestamp;
}

public String getPropertyName() {
Expand All @@ -84,4 +99,8 @@ public String getPropertyName() {
public boolean isCredential() {
return isCredential;
}

public boolean isExpirationTimestamp() {
return isExpirationTimestamp;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,13 @@

import jakarta.annotation.Nonnull;
import java.net.URI;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Stream;
import org.apache.polaris.core.context.CallContext;
import org.apache.polaris.core.storage.AccessConfig;
import org.apache.polaris.core.storage.InMemoryStorageIntegration;
import org.apache.polaris.core.storage.StorageAccessProperty;
import org.apache.polaris.core.storage.StorageUtil;
Expand Down Expand Up @@ -66,7 +66,7 @@ public AwsCredentialsStorageIntegration(

/** {@inheritDoc} */
@Override
public EnumMap<StorageAccessProperty, String> getSubscopedCreds(
public AccessConfig getSubscopedCreds(
@Nonnull CallContext callContext,
@Nonnull AwsStorageConfigurationInfo storageConfig,
boolean allowListOperation,
Expand All @@ -90,50 +90,48 @@ public EnumMap<StorageAccessProperty, String> getSubscopedCreds(
credentialsProvider.ifPresent(
cp -> request.overrideConfiguration(b -> b.credentialsProvider(cp)));

String region = storageConfig.getRegion();
@SuppressWarnings("resource")
// Note: stsClientProvider returns "thin" clients that do not need closing
StsClient stsClient =
stsClientProvider.stsClient(
StsDestination.of(storageConfig.getStsEndpointUri(), storageConfig.getRegion()));
stsClientProvider.stsClient(StsDestination.of(storageConfig.getStsEndpointUri(), region));

AssumeRoleResponse response = stsClient.assumeRole(request.build());
EnumMap<StorageAccessProperty, String> credentialMap =
new EnumMap<>(StorageAccessProperty.class);
credentialMap.put(StorageAccessProperty.AWS_KEY_ID, response.credentials().accessKeyId());
credentialMap.put(
AccessConfig.Builder accessConfig = AccessConfig.builder();
accessConfig.put(StorageAccessProperty.AWS_KEY_ID, response.credentials().accessKeyId());
accessConfig.put(
StorageAccessProperty.AWS_SECRET_KEY, response.credentials().secretAccessKey());
credentialMap.put(StorageAccessProperty.AWS_TOKEN, response.credentials().sessionToken());
accessConfig.put(StorageAccessProperty.AWS_TOKEN, response.credentials().sessionToken());
Optional.ofNullable(response.credentials().expiration())
.ifPresent(
i -> {
credentialMap.put(
accessConfig.put(
StorageAccessProperty.EXPIRATION_TIME, String.valueOf(i.toEpochMilli()));
credentialMap.put(
accessConfig.put(
StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS,
String.valueOf(i.toEpochMilli()));
});

if (storageConfig.getRegion() != null) {
credentialMap.put(StorageAccessProperty.CLIENT_REGION, storageConfig.getRegion());
if (region != null) {
accessConfig.put(StorageAccessProperty.CLIENT_REGION, region);
}

URI endpointUri = storageConfig.getEndpointUri();
if (endpointUri != null) {
credentialMap.put(StorageAccessProperty.AWS_ENDPOINT, endpointUri.toString());
accessConfig.put(StorageAccessProperty.AWS_ENDPOINT, endpointUri.toString());
}

if (Boolean.TRUE.equals(storageConfig.getPathStyleAccess())) {
credentialMap.put(StorageAccessProperty.AWS_PATH_STYLE_ACCESS, Boolean.TRUE.toString());
accessConfig.put(StorageAccessProperty.AWS_PATH_STYLE_ACCESS, Boolean.TRUE.toString());
}

if (storageConfig.getAwsPartition().equals("aws-us-gov")
&& credentialMap.get(StorageAccessProperty.CLIENT_REGION) == null) {
if (storageConfig.getAwsPartition().equals("aws-us-gov") && region == null) {
throw new IllegalArgumentException(
String.format(
"AWS region must be set when using partition %s", storageConfig.getAwsPartition()));
}

return credentialMap;
return accessConfig.build();
}

/**
Expand Down
Loading