From c13010667524c99d695cf6cc8d87a3a0957def1a Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Thu, 31 Jul 2025 15:09:15 -0700 Subject: [PATCH 1/2] Rest: Refresh Creds for long running tasks --- .../iceberg/rest/RESTSessionCatalog.java | 20 ++++++++++++++++++- .../apache/iceberg/rest/ResourcePaths.java | 11 ++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java index 4e1c339d1fe9..87a35aefa9cf 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java @@ -447,7 +447,7 @@ public Table loadTable(SessionContext context, TableIdentifier identifier) { tableClient, paths.table(finalIdentifier), Map::of, - tableFileIO(context, tableConf, response.credentials()), + tableFileIO(context, identifier, tableConf, response.credentials()), tableMetadata, endpoints); @@ -992,6 +992,24 @@ private FileIO newFileIO( } } + private FileIO tableFileIO( + SessionContext context, + TableIdentifier identifier, + Map config, + List storageCredentials) { + // inject the credentials refresh endpoint if, refresh is configured + // TODO: convert this to constants. + boolean refreshEnabled = + PropertyUtil.propertyAsBoolean(config, "client.refresh-credentials-enabled", false); + if (refreshEnabled && endpoints.contains(Endpoint.V1_TABLE_CREDENTIALS)) { + config.put("client.refresh-credentials-endpoint", paths.tableCredentials(identifier)); + } + + // FileIO will use the credential provider which will use the refresh endpoint when + // the vended credentials from load table are close to expiration. + return tableFileIO(context, config, storageCredentials); + } + private FileIO tableFileIO( SessionContext context, Map config, List storageCredentials) { if (config.isEmpty() && ioBuilder == null && storageCredentials.isEmpty()) { diff --git a/core/src/main/java/org/apache/iceberg/rest/ResourcePaths.java b/core/src/main/java/org/apache/iceberg/rest/ResourcePaths.java index d5c11f6052f1..1d57b2cc483f 100644 --- a/core/src/main/java/org/apache/iceberg/rest/ResourcePaths.java +++ b/core/src/main/java/org/apache/iceberg/rest/ResourcePaths.java @@ -90,6 +90,17 @@ public String table(TableIdentifier ident) { RESTUtil.encodeString(ident.name())); } + public String tableCredentials(TableIdentifier ident) { + return SLASH.join( + "v1", + prefix, + "namespaces", + RESTUtil.encodeNamespace(ident.namespace()), + "tables", + RESTUtil.encodeString(ident.name()), + "credentials"); + } + public String register(Namespace ns) { return SLASH.join("v1", prefix, "namespaces", RESTUtil.encodeNamespace(ns), "register"); } From d2a07c0cd2b57803118649597d41181e6d6ac9e8 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sun, 3 Aug 2025 08:32:13 -0700 Subject: [PATCH 2/2] review feedback --- .../iceberg/rest/RESTSessionCatalog.java | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java index 87a35aefa9cf..a2fdadb61a7f 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java @@ -997,12 +997,26 @@ private FileIO tableFileIO( TableIdentifier identifier, Map config, List storageCredentials) { - // inject the credentials refresh endpoint if, refresh is configured + // Check if either credential refresh is enabled from either client side or server side. // TODO: convert this to constants. - boolean refreshEnabled = - PropertyUtil.propertyAsBoolean(config, "client.refresh-credentials-enabled", false); - if (refreshEnabled && endpoints.contains(Endpoint.V1_TABLE_CREDENTIALS)) { - config.put("client.refresh-credentials-endpoint", paths.tableCredentials(identifier)); + boolean s3RefreshEnabled = + PropertyUtil.propertyAsBoolean(config, "client.refresh-credentials-enabled", false) + || PropertyUtil.propertyAsBoolean( + properties(), "client.refresh-credentials-enabled", false); + boolean adlsRefreshEnabled = + PropertyUtil.propertyAsBoolean(config, "adls.refresh-credentials-enabled", false) + || PropertyUtil.propertyAsBoolean( + properties(), "adls.refresh-credentials-enabled", false); + + // Inject the credentials refresh endpoint if, refresh is configured. + // This is done because the server would not know the complete URI of the refresh endpoint. + // The IRC relative path for refreshing the credentials for a table is fixed. + if (s3RefreshEnabled && endpoints.contains(Endpoint.V1_TABLE_CREDENTIALS)) { + // respect server-side refresh endpoint configuration. + config.putIfAbsent("client.refresh-credentials-endpoint", paths.tableCredentials(identifier)); + } else if (adlsRefreshEnabled && endpoints.contains(Endpoint.V1_TABLE_CREDENTIALS)) { + // respect server-side refresh endpoint configuration. + config.putIfAbsent("adls.refresh-credentials-endpoint", paths.tableCredentials(identifier)); } // FileIO will use the credential provider which will use the refresh endpoint when