[apache#6394] feat(iceberg): upgrade Iceberg version from 1.5.2 to 1.…

…6.1 (apache#6374)

### What changes were proposed in this pull request?

Upgrade Iceberg version from 1.5.2 to 1.6.1

In 1.6, the `token` endpoint is deprecated and there're no new endpoint
is added.


### Why are the changes needed?

Fix: apache#6394 

### Does this PR introduce _any_ user-facing change?
no

### How was this patch tested?
run pass existing ITs

build.gradle.kts

@@ -191,6 +191,9 @@ allprojects {
         param.environment("NEED_CREATE_DOCKER_NETWORK", "true")
       }
 
+      val icebergVersion: String = libs.versions.iceberg.get()
+      param.systemProperty("ICEBERG_VERSION", icebergVersion)
+
       // Change poll image pause time from 30s to 60s
       param.environment("TESTCONTAINERS_PULL_PAUSE_TIMEOUT", "60")
       val jdbcDatabase = project.properties["jdbcBackend"] as? String ?: "h2"

dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh

@@ -49,19 +49,21 @@ cp ${gravitino_home}/bundles/aws/build/libs/gravitino-aws-*.jar bundles/
 cp ${gravitino_home}/bundles/azure/build/libs/gravitino-azure-*.jar bundles/
 cp ${gravitino_home}/bundles/aliyun-bundle/build/libs/gravitino-aliyun-bundle-*.jar bundles/
 
-iceberg_gcp_bundle="iceberg-gcp-bundle-1.5.2.jar"
+iceberg_version="1.6.1"
+
+iceberg_gcp_bundle="iceberg-gcp-bundle-${iceberg_version}.jar"
 if [ ! -f "bundles/${iceberg_gcp_bundle}" ]; then
-  curl -L -s -o bundles/${iceberg_gcp_bundle} https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-gcp-bundle/1.5.2/${iceberg_gcp_bundle}
+  curl -L -s -o bundles/${iceberg_gcp_bundle} https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-gcp-bundle/${iceberg_version}/${iceberg_gcp_bundle}
 fi
 
-iceberg_aws_bundle="iceberg-aws-bundle-1.5.2.jar"
+iceberg_aws_bundle="iceberg-aws-bundle-${iceberg_version}.jar"
 if [ ! -f "bundles/${iceberg_aws_bundle}" ]; then
-  curl -L -s -o bundles/${iceberg_aws_bundle} https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/1.5.2/${iceberg_aws_bundle}
+  curl -L -s -o bundles/${iceberg_aws_bundle} https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${iceberg_version}/${iceberg_aws_bundle}
 fi
 
-iceberg_azure_bundle="iceberg-azure-bundle-1.5.2.jar"
+iceberg_azure_bundle="iceberg-azure-bundle-${iceberg_version}.jar"
 if [ ! -f "bundles/${iceberg_azure_bundle}" ]; then
-  curl -L -s -o bundles/${iceberg_azure_bundle} https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-azure-bundle/1.5.2/${iceberg_azure_bundle}
+  curl -L -s -o bundles/${iceberg_azure_bundle} https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-azure-bundle/${iceberg_version}/${iceberg_azure_bundle}
 fi
 
 # download jdbc driver

docs/iceberg-rest-service.md

@@ -13,7 +13,6 @@ The Apache Gravitino Iceberg REST Server follows the [Apache Iceberg REST API sp
 ### Capabilities
 
 - Supports the Apache Iceberg REST API defined in Iceberg 1.5, and supports all namespace and table interfaces. The following interfaces are not implemented yet:
-  - token
   - multi table transaction
   - pagination
 - Works as a catalog proxy, supporting `Hive` and `JDBC` as catalog backend.
@@ -317,7 +316,7 @@ gravitino.iceberg-rest.gravitino-metalake = test
 
 ### Other Apache Iceberg catalog properties
 
-You can add other properties defined in [Iceberg catalog properties](https://iceberg.apache.org/docs/1.5.2/configuration/#catalog-properties).
+You can add other properties defined in [Iceberg catalog properties](https://iceberg.apache.org/docs/1.6.1/configuration/#catalog-properties).
 The `clients` property for example:
 
 | Configuration item               | Description                          | Default value | Required |
@@ -379,7 +378,7 @@ Normally you will see the output like `{"defaults":{},"overrides":{}}%`.
 
 ### Deploying Apache Spark with Apache Iceberg support
 
-Follow the [Spark Iceberg start guide](https://iceberg.apache.org/docs/1.5.2/spark-getting-started/) to set up Apache Spark's and Apache Iceberg's environment.
+Follow the [Spark Iceberg start guide](https://iceberg.apache.org/docs/1.6.1/spark-getting-started/) to set up Apache Spark's and Apache Iceberg's environment.
 
 ### Starting the Apache Spark client with the Apache Iceberg REST catalog
 

docs/lakehouse-iceberg-catalog.md

@@ -18,7 +18,7 @@ Apache Gravitino provides the ability to manage Apache Iceberg metadata.
 ### Requirements and limitations
 
 :::info
-Builds with Apache Iceberg `1.5.2`. The Apache Iceberg table format version is `2` by default.
+Builds with Apache Iceberg `1.6.1`. The Apache Iceberg table format version is `2` by default.
 :::
 
 ## Catalog

gradle/libs.versions.toml

@@ -55,7 +55,7 @@ commons-collections3 = "3.2.2"
 commons-configuration1 = "1.6"
 commons-dbcp2 = "2.11.0"
 caffeine = "2.9.3"
-iceberg = '1.5.2' # used for Gravitino Iceberg catalog and Iceberg REST service
+iceberg = '1.6.1' # used for Gravitino Iceberg catalog and Iceberg REST service
 iceberg4spark = "1.4.1" # used for compile spark connector
 paimon = '0.8.0'
 spark33 = "3.3.4"

iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java

@@ -117,11 +117,11 @@ private Map<String, String> getADLSConfig() {
   }
 
   private void downloadIcebergAzureBundleJar() throws IOException {
-    String icebergBundleJarName = "iceberg-azure-bundle-1.5.2.jar";
     String icebergBundleJarUri =
-        "https://repo1.maven.org/maven2/org/apache/iceberg/"
-            + "iceberg-azure-bundle/1.5.2/"
-            + icebergBundleJarName;
+        String.format(
+            "https://repo1.maven.org/maven2/org/apache/iceberg/"
+                + "iceberg-azure-bundle/%s/iceberg-azure-bundle-%s.jar",
+            ITUtils.icebergVersion(), ITUtils.icebergVersion());
     String gravitinoHome = System.getenv("GRAVITINO_HOME");
     String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome);
     DownloaderUtils.downloadFile(icebergBundleJarUri, targetDir);

iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java

@@ -100,11 +100,11 @@ private Map<String, String> getADLSConfig() {
   }
 
   private void downloadIcebergAzureBundleJar() throws IOException {
-    String icebergBundleJarName = "iceberg-azure-bundle-1.5.2.jar";
     String icebergBundleJarUri =
-        "https://repo1.maven.org/maven2/org/apache/iceberg/"
-            + "iceberg-azure-bundle/1.5.2/"
-            + icebergBundleJarName;
+        String.format(
+            "https://repo1.maven.org/maven2/org/apache/iceberg/"
+                + "iceberg-azure-bundle/%s/iceberg-azure-bundle-%s.jar",
+            ITUtils.icebergVersion(), ITUtils.icebergVersion());
     String gravitinoHome = System.getenv("GRAVITINO_HOME");
     String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome);
     DownloaderUtils.downloadFile(icebergBundleJarUri, targetDir);

iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java

@@ -35,6 +35,7 @@
 // You should export GRAVITINO_GCS_BUCKET and GOOGLE_APPLICATION_CREDENTIALS to run the test
 @EnabledIfEnvironmentVariable(named = "GRAVITINO_TEST_CLOUD_IT", matches = "true")
 public class IcebergRESTGCSIT extends IcebergRESTJdbcCatalogIT {
+
   private String gcsWarehouse;
   private String gcsCredentialPath;
 
@@ -92,10 +93,11 @@ private void copyGCSBundleJar() {
   }
 
   private void downloadIcebergBundleJar() throws IOException {
-    String icebergBundleJarName = "iceberg-gcp-bundle-1.5.2.jar";
     String icebergBundleJarUri =
-        "https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-gcp-bundle/1.5.2/"
-            + icebergBundleJarName;
+        String.format(
+            "https://repo1.maven.org/maven2/org/apache/iceberg/"
+                + "iceberg-gcp-bundle/%s/iceberg-gcp-bundle-%s.jar",
+            ITUtils.icebergVersion(), ITUtils.icebergVersion());
     String gravitinoHome = System.getenv("GRAVITINO_HOME");
     String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome);
     DownloaderUtils.downloadFile(icebergBundleJarUri, targetDir);

iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java

@@ -113,11 +113,11 @@ private Map<String, String> getOSSConfig() {
   }
 
   private void downloadIcebergForAliyunJar() throws IOException {
-    String icebergBundleJarName = "iceberg-aliyun-1.5.2.jar";
     String icebergBundleJarUri =
-        "https://repo1.maven.org/maven2/org/apache/iceberg/"
-            + "iceberg-aliyun/1.5.2/"
-            + icebergBundleJarName;
+        String.format(
+            "https://repo1.maven.org/maven2/org/apache/iceberg/"
+                + "iceberg-aliyun/%s/iceberg-aliyun-%s.jar",
+            ITUtils.icebergVersion(), ITUtils.icebergVersion());
     String gravitinoHome = System.getenv("GRAVITINO_HOME");
     String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome);
     DownloaderUtils.downloadFile(icebergBundleJarUri, targetDir);

iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java

@@ -98,11 +98,11 @@ private Map<String, String> getOSSConfig() {
   }
 
   private void downloadIcebergForAliyunJar() throws IOException {
-    String icebergBundleJarName = "iceberg-aliyun-1.5.2.jar";
     String icebergBundleJarUri =
-        "https://repo1.maven.org/maven2/org/apache/iceberg/"
-            + "iceberg-aliyun/1.5.2/"
-            + icebergBundleJarName;
+        String.format(
+            "https://repo1.maven.org/maven2/org/apache/iceberg/"
+                + "iceberg-aliyun/%s/iceberg-aliyun-%s.jar",
+            ITUtils.icebergVersion(), ITUtils.icebergVersion());
     String gravitinoHome = System.getenv("GRAVITINO_HOME");
     String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome);
     DownloaderUtils.downloadFile(icebergBundleJarUri, targetDir);

iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenIT.java

@@ -111,11 +111,11 @@ private Map<String, String> getS3Config() {
   }
 
   private void downloadIcebergAwsBundleJar() throws IOException {
-    String icebergBundleJarName = "iceberg-aws-bundle-1.5.2.jar";
     String icebergBundleJarUri =
-        "https://repo1.maven.org/maven2/org/apache/iceberg/"
-            + "iceberg-aws-bundle/1.5.2/"
-            + icebergBundleJarName;
+        String.format(
+            "https://repo1.maven.org/maven2/org/apache/iceberg/"
+                + "iceberg-aws-bundle/%s/iceberg-aws-bundle-%s.jar",
+            ITUtils.icebergVersion(), ITUtils.icebergVersion());
     String gravitinoHome = System.getenv("GRAVITINO_HOME");
     String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome);
     DownloaderUtils.downloadFile(icebergBundleJarUri, targetDir);

integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/ITUtils.java

@@ -187,6 +187,10 @@ public static boolean isEmbedded() {
     return Objects.equals(mode, ITUtils.EMBEDDED_TEST_MODE);
   }
 
+  public static String icebergVersion() {
+    return System.getProperty("ICEBERG_VERSION");
+  }
+
   public static String getBundleJarSourceFile(String bundleName) {
     String jarName = ITUtils.getBundleJarName(bundleName);
     String gcsJars = ITUtils.joinPath(ITUtils.getBundleJarDirectory(bundleName), jarName);

trino-connector/integration-test/trino-test-tools/download_jar.sh

@@ -62,8 +62,8 @@ download_postgresql_jar() {
 }
 
 download_iceberg_aws_bundle() {
-  download_jar "iceberg-aws-bundle--1.5.2.jar" \
-  "https://jdbc.postgresql.org/download/iceberg-aws-bundle--1.5.2.jar" \
+  download_jar "iceberg-aws-bundle-1.6.1.jar" \
+  "https://jdbc.postgresql.org/download/iceberg-aws-bundle-1.6.1.jar" \
   "$GRAVITINO_SERVER_DIR/catalogs/lakehouse-iceberg/libs"
 }