From 9f35abfa9a95fbf25ecdb5d169ca841dc38f5221 Mon Sep 17 00:00:00 2001 From: Ray Liu Date: Thu, 23 Jul 2026 17:05:41 +0800 Subject: [PATCH 1/8] Fix double escaping of Iceberg S3 input-file URIs Signed-off-by: Ray Liu --- .../org/apache/iceberg/aws/s3/IcebergS3InputFileAccess.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/iceberg/common/src/main/java/org/apache/iceberg/aws/s3/IcebergS3InputFileAccess.java b/iceberg/common/src/main/java/org/apache/iceberg/aws/s3/IcebergS3InputFileAccess.java index 3cd9c7c1bff..db4dd923b0b 100644 --- a/iceberg/common/src/main/java/org/apache/iceberg/aws/s3/IcebergS3InputFileAccess.java +++ b/iceberg/common/src/main/java/org/apache/iceberg/aws/s3/IcebergS3InputFileAccess.java @@ -32,7 +32,10 @@ public static URI s3Uri(InputFile inputFile) { } S3URI uri = ((BaseS3File) inputFile).uri(); try { - return new URI("s3", uri.bucket(), "/" + uri.key(), null); + // S3URI.key() is taken from the original location and may already contain + // URL escapes. Parse the complete URI rather than treating its key as a + // URI component, which would escape '%' a second time. + return new URI("s3://" + uri.bucket() + "/" + uri.key()); } catch (URISyntaxException e) { throw new IllegalArgumentException( "Invalid S3 URI for bucket=" + uri.bucket() + " key=" + uri.key(), e); From 8048550f904a96aa48b6249ae6da2dd6f6a86e84 Mon Sep 17 00:00:00 2001 From: Ray Liu Date: Fri, 24 Jul 2026 15:21:36 +0800 Subject: [PATCH 2/8] Address comments --- dist/unshimmed-common-from-single-shim.txt | 1 - .../iceberg/aws/s3/IcebergS3InputFile.java | 34 ++++++++------ .../aws/s3/IcebergS3InputFileAccess.java | 44 ------------------- .../src/main/python/iceberg/iceberg_test.py | 28 +++++++++++- 4 files changed, 47 insertions(+), 60 deletions(-) delete mode 100644 iceberg/common/src/main/java/org/apache/iceberg/aws/s3/IcebergS3InputFileAccess.java diff --git a/dist/unshimmed-common-from-single-shim.txt b/dist/unshimmed-common-from-single-shim.txt index 6b3a53fa176..3cef57f7022 100644 --- a/dist/unshimmed-common-from-single-shim.txt +++ b/dist/unshimmed-common-from-single-shim.txt @@ -30,7 +30,6 @@ com/nvidia/spark/rapids/iceberg/spark/GpuSparkUtil$.class com/nvidia/spark/rapids/iceberg/spark/RapidsSparkCatalog.class com/nvidia/spark/rapids/iceberg/spark/RapidsSparkSessionCatalog.class com/nvidia/spark/rapids/iceberg/spark/source/RapidsSparkTable.class -org/apache/iceberg/aws/s3/IcebergS3InputFileAccess.class org/apache/iceberg/data/GpuFileHelpers.class org/apache/iceberg/io/GpuClusteredWriterBridge.class org/apache/iceberg/io/GpuFanoutWriterBridge.class diff --git a/iceberg/common/src/main/java/org/apache/iceberg/aws/s3/IcebergS3InputFile.java b/iceberg/common/src/main/java/org/apache/iceberg/aws/s3/IcebergS3InputFile.java index 7bebf280c6e..e54bce8328f 100644 --- a/iceberg/common/src/main/java/org/apache/iceberg/aws/s3/IcebergS3InputFile.java +++ b/iceberg/common/src/main/java/org/apache/iceberg/aws/s3/IcebergS3InputFile.java @@ -32,7 +32,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.net.URI; import java.util.List; import java.util.OptionalLong; @@ -40,20 +39,23 @@ * S3-backed {@link RapidsInputFile} that delegates byte-range reads to * {@link IcebergS3RangeCopier}. The supplied {@link FileIO} is only used for * its property map and any per-prefix storage-credential overlays. - * - *

The package-private S3 file access is isolated in {@link IcebergS3InputFileAccess}. */ public final class IcebergS3InputFile implements RapidsInputFile { private static final Logger LOG = LoggerFactory.getLogger(IcebergS3InputFile.class); private final IcebergInputFile delegate; - private final URI s3Uri; + private final String s3Bucket; + private final String s3Key; private final IcebergS3Client icebergS3Client; private IcebergS3InputFile( - IcebergInputFile delegate, URI s3Uri, IcebergS3Client icebergS3Client) { + IcebergInputFile delegate, + String s3Bucket, + String s3Key, + IcebergS3Client icebergS3Client) { this.delegate = delegate; - this.s3Uri = s3Uri; + this.s3Bucket = s3Bucket; + this.s3Key = s3Key; this.icebergS3Client = icebergS3Client; } @@ -64,25 +66,27 @@ public static RapidsInputFile maybeCreate(InputFile inputFile, FileIO fileIO) { if (!RapidsInputFiles.isS3PerfEnabled()) { return delegate; } - URI s3Uri = IcebergS3InputFileAccess.s3Uri(inputFile); - if (s3Uri == null) { + if (!(inputFile instanceof BaseS3File)) { return delegate; } + S3URI s3Uri = ((BaseS3File) inputFile).uri(); + String s3Bucket = s3Uri.bucket(); + String s3Key = s3Uri.key(); // Iceberg < 1.7 does not have SupportsStorageCredentials; ShimUtils returns // the per-prefix credential overlays (or an empty map on 1.6). IcebergS3Client icebergS3Client = IcebergS3RangeCopier.resolveClient( - s3Uri.toString(), + inputFile.location(), fileIO.properties(), ShimUtils.storageCredentialOverlays(fileIO)); if (icebergS3Client == null) { if (TaskContext.get() != null) { GpuTaskMetrics$.MODULE$.get().recordPerfioS3IcebergFallback(); } - LOG.debug("IcebergS3RangeCopier path disabled for {}", s3Uri); + LOG.debug("IcebergS3RangeCopier path disabled for {}", inputFile.location()); return delegate; } - LOG.debug("IcebergS3RangeCopier path active for {}", s3Uri); - return new IcebergS3InputFile(delegate, s3Uri, icebergS3Client); + LOG.debug("IcebergS3RangeCopier path active for {}", inputFile.location()); + return new IcebergS3InputFile(delegate, s3Bucket, s3Key, icebergS3Client); } @Override @@ -117,7 +121,8 @@ public InputFile getDelegate() { @Override public void readVectored(HostMemoryBuffer output, List copyRanges) throws IOException { - IcebergS3RangeCopier.copyToHMB(icebergS3Client, output, s3Uri, copyRanges); + IcebergS3RangeCopier.copyToHMB( + icebergS3Client, output, s3Bucket, s3Key, copyRanges); } /** @@ -133,6 +138,7 @@ public void readTail(long length, HostMemoryBuffer output) throws IOException { if (length < 0) { throw new IllegalArgumentException("length must be non-negative"); } - IcebergS3RangeCopier.copyTailToHMB(icebergS3Client, output, s3Uri, length, /*dstOffset*/ 0L); + IcebergS3RangeCopier.copyTailToHMB( + icebergS3Client, output, s3Bucket, s3Key, length, /*dstOffset*/ 0L); } } diff --git a/iceberg/common/src/main/java/org/apache/iceberg/aws/s3/IcebergS3InputFileAccess.java b/iceberg/common/src/main/java/org/apache/iceberg/aws/s3/IcebergS3InputFileAccess.java deleted file mode 100644 index db4dd923b0b..00000000000 --- a/iceberg/common/src/main/java/org/apache/iceberg/aws/s3/IcebergS3InputFileAccess.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2026, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.iceberg.aws.s3; - -import java.net.URI; -import java.net.URISyntaxException; - -import org.apache.iceberg.io.InputFile; - -/** Package-local access to Iceberg S3 file internals. */ -public final class IcebergS3InputFileAccess { - private IcebergS3InputFileAccess() { - } - - public static URI s3Uri(InputFile inputFile) { - if (!(inputFile instanceof BaseS3File)) { - return null; - } - S3URI uri = ((BaseS3File) inputFile).uri(); - try { - // S3URI.key() is taken from the original location and may already contain - // URL escapes. Parse the complete URI rather than treating its key as a - // URI component, which would escape '%' a second time. - return new URI("s3://" + uri.bucket() + "/" + uri.key()); - } catch (URISyntaxException e) { - throw new IllegalArgumentException( - "Invalid S3 URI for bucket=" + uri.bucket() + " key=" + uri.key(), e); - } - } -} diff --git a/integration_tests/src/main/python/iceberg/iceberg_test.py b/integration_tests/src/main/python/iceberg/iceberg_test.py index 87345b108d1..c7727e7aeeb 100644 --- a/integration_tests/src/main/python/iceberg/iceberg_test.py +++ b/integration_tests/src/main/python/iceberg/iceberg_test.py @@ -15,7 +15,7 @@ import pytest from asserts import assert_equal_with_local_sort, assert_gpu_and_cpu_are_equal_collect, assert_gpu_and_cpu_row_counts_equal, assert_gpu_fallback_collect, assert_spark_exception -from conftest import is_iceberg_remote_catalog +from conftest import is_iceberg_remote_catalog, is_iceberg_rest_catalog from data_gen import * from iceberg import get_full_table_name, iceberg_unsupported_mark, _build_tblprops, \ _BASE_TBLPROPS_SQL, create_iceberg_table @@ -629,6 +629,32 @@ def setup_iceberg_table(spark): lambda spark: spark.sql("SELECT * FROM {}".format(table)), conf={'spark.rapids.sql.format.parquet.reader.type': reader_type}) +@iceberg +@ignore_order(local=True) # Iceberg plans with a thread pool and is not deterministic in file ordering +@pytest.mark.parametrize('reader_type', rapids_reader_types) +@pytest.mark.skipif(not is_iceberg_rest_catalog(), + reason="S3 path handling is exercised only with the REST catalog") +def test_iceberg_parquet_read_from_uri_invalid_s3_path(spark_tmp_table_factory, reader_type): + table = get_full_table_name(spark_tmp_table_factory) + tmp_view = spark_tmp_table_factory.get() + + def setup_iceberg_table(spark): + # A raw space is valid in an S3 object key but invalid in a URI. The RAPIDS reader must + # retain Iceberg's original key for the S3 request instead of using a URI-encoded path. + warehouse = spark.conf.get('spark.sql.catalog.spark_catalog.warehouse').rstrip('/') + data_path = f'{warehouse}/{spark_tmp_table_factory.get()} uri invalid path/data' + df = two_col_df(spark, long_gen, string_gen).sortWithinPartitions('b') + df.createOrReplaceTempView(tmp_view) + props = _build_tblprops({'write.data.path': data_path}) + props_sql = ", ".join(f"'{k}' = '{v}'" for k, v in props.items()) + spark.sql(f"CREATE TABLE {table} USING ICEBERG TBLPROPERTIES ({props_sql}) " + f"AS SELECT * FROM {tmp_view}") + + with_cpu_session(setup_iceberg_table) + assert_gpu_and_cpu_are_equal_collect( + lambda spark: spark.sql(f"SELECT * FROM {table}"), + conf={'spark.rapids.sql.format.parquet.reader.type': reader_type}) + @iceberg @ignore_order(local=True) # Iceberg plans with a thread pool and is not deterministic in file ordering @pytest.mark.parametrize('reader_type', rapids_reader_types) From 42d44d53254be9d16e3e14bafd1bbad02f57fa1e Mon Sep 17 00:00:00 2001 From: Ray Liu Date: Mon, 27 Jul 2026 11:57:03 +0800 Subject: [PATCH 3/8] Fix build break --- .../apache/spark/sql/rapids/shims/CreateNamedStructShims.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/CreateNamedStructShims.scala b/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/CreateNamedStructShims.scala index d8838944380..7af931d9df1 100644 --- a/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/CreateNamedStructShims.scala +++ b/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/CreateNamedStructShims.scala @@ -37,6 +37,7 @@ {"spark": "356"} {"spark": "357"} {"spark": "358"} +{"spark": "359"} {"spark": "400"} {"spark": "400db173"} {"spark": "401"} From bb08b45a9bf2a8e216de4dc9686d0af368619500 Mon Sep 17 00:00:00 2001 From: Ray Liu Date: Mon, 27 Jul 2026 14:50:59 +0800 Subject: [PATCH 4/8] Add conf to ensure perf io is enabled --- integration_tests/src/main/python/iceberg/iceberg_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/integration_tests/src/main/python/iceberg/iceberg_test.py b/integration_tests/src/main/python/iceberg/iceberg_test.py index c7727e7aeeb..05062919171 100644 --- a/integration_tests/src/main/python/iceberg/iceberg_test.py +++ b/integration_tests/src/main/python/iceberg/iceberg_test.py @@ -653,7 +653,10 @@ def setup_iceberg_table(spark): with_cpu_session(setup_iceberg_table) assert_gpu_and_cpu_are_equal_collect( lambda spark: spark.sql(f"SELECT * FROM {table}"), - conf={'spark.rapids.sql.format.parquet.reader.type': reader_type}) + conf={ + 'spark.rapids.perfio.s3.enabled': 'true', + 'spark.rapids.sql.format.parquet.reader.type': reader_type, + }) @iceberg @ignore_order(local=True) # Iceberg plans with a thread pool and is not deterministic in file ordering From 8a11bab700ed288c7f6c029158a130887b62ce02 Mon Sep 17 00:00:00 2001 From: Ray Liu Date: Mon, 27 Jul 2026 17:35:13 +0800 Subject: [PATCH 5/8] Address comments --- .../src/main/python/iceberg/iceberg_test.py | 9 +++++---- jenkins/spark-tests.sh | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/integration_tests/src/main/python/iceberg/iceberg_test.py b/integration_tests/src/main/python/iceberg/iceberg_test.py index 05062919171..465456d1151 100644 --- a/integration_tests/src/main/python/iceberg/iceberg_test.py +++ b/integration_tests/src/main/python/iceberg/iceberg_test.py @@ -651,12 +651,13 @@ def setup_iceberg_table(spark): f"AS SELECT * FROM {tmp_view}") with_cpu_session(setup_iceberg_table) + assert with_gpu_session( + lambda spark: + spark._jvm.com.nvidia.spark.rapids.fileio.RapidsInputFiles.isS3PerfEnabled()), \ + "PerfIO S3 must be enabled at Spark startup for REST catalog tests" assert_gpu_and_cpu_are_equal_collect( lambda spark: spark.sql(f"SELECT * FROM {table}"), - conf={ - 'spark.rapids.perfio.s3.enabled': 'true', - 'spark.rapids.sql.format.parquet.reader.type': reader_type, - }) + conf={'spark.rapids.sql.format.parquet.reader.type': reader_type}) @iceberg @ignore_order(local=True) # Iceberg plans with a thread pool and is not deterministic in file ordering diff --git a/jenkins/spark-tests.sh b/jenkins/spark-tests.sh index 1777bdc6b31..a1e04216de9 100755 --- a/jenkins/spark-tests.sh +++ b/jenkins/spark-tests.sh @@ -347,9 +347,8 @@ run_iceberg_tests() { echo "!!! Running iceberg tests with rest catalog" ICEBERG_REST_JARS="org.apache.iceberg:iceberg-spark-runtime-${ICEBERG_SPARK_VER}_${SCALA_BINARY_VER}:${ICEBERG_VERSION},\ org.apache.iceberg:iceberg-aws-bundle:${ICEBERG_VERSION}" - # filecache.enabled is a startup-only config, so it must be set here via - # PYSP_TEST_ env var rather than as a session-level Spark config, because - # FileCacheManager is initialized at executor startup time. + # filecache.enabled and perfio.s3.enabled are startup-only configs, so they must + # be set here via PYSP_TEST_ env vars rather than as session-level Spark configs. env \ HOST_NAME=$PROJECT_REPO_HOST \ EXPECTED_ICEBERG_VERSION=${ICEBERG_VERSION} \ @@ -358,6 +357,7 @@ org.apache.iceberg:iceberg-aws-bundle:${ICEBERG_VERSION}" PYSP_TEST_spark_driver_memory=1G \ PYSP_TEST_spark_executor_memory=2G \ PYSP_TEST_spark_rapids_filecache_enabled=true \ + PYSP_TEST_spark_rapids_perfio_s3_enabled=true \ PYSP_TEST_spark_jars_packages="${ICEBERG_REST_JARS}" \ PYSP_TEST_spark_jars_ivySettings="${WORKSPACE}/jenkins/ivysettings.xml" \ PYSP_TEST_spark_sql_extensions="org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions" \ From 58e24ed09b69d0a30487d20f737dacbb71ac321e Mon Sep 17 00:00:00 2001 From: Ray Liu Date: Wed, 29 Jul 2026 16:51:38 +0800 Subject: [PATCH 6/8] new use case --- .../src/main/python/iceberg/iceberg_test.py | 14 ++---- jenkins/spark-tests.sh | 47 ++++++++++++++----- 2 files changed, 41 insertions(+), 20 deletions(-) diff --git a/integration_tests/src/main/python/iceberg/iceberg_test.py b/integration_tests/src/main/python/iceberg/iceberg_test.py index 465456d1151..d752266accd 100644 --- a/integration_tests/src/main/python/iceberg/iceberg_test.py +++ b/integration_tests/src/main/python/iceberg/iceberg_test.py @@ -637,18 +637,14 @@ def setup_iceberg_table(spark): def test_iceberg_parquet_read_from_uri_invalid_s3_path(spark_tmp_table_factory, reader_type): table = get_full_table_name(spark_tmp_table_factory) tmp_view = spark_tmp_table_factory.get() + partition_gen = StringGen(pattern="(.|\n){1,10}", nullable=False)\ + .with_special_case('uri invalid path', 1000) def setup_iceberg_table(spark): - # A raw space is valid in an S3 object key but invalid in a URI. The RAPIDS reader must - # retain Iceberg's original key for the S3 request instead of using a URI-encoded path. - warehouse = spark.conf.get('spark.sql.catalog.spark_catalog.warehouse').rstrip('/') - data_path = f'{warehouse}/{spark_tmp_table_factory.get()} uri invalid path/data' - df = two_col_df(spark, long_gen, string_gen).sortWithinPartitions('b') + df = two_col_df(spark, long_gen, partition_gen).sortWithinPartitions('b') df.createOrReplaceTempView(tmp_view) - props = _build_tblprops({'write.data.path': data_path}) - props_sql = ", ".join(f"'{k}' = '{v}'" for k, v in props.items()) - spark.sql(f"CREATE TABLE {table} USING ICEBERG TBLPROPERTIES ({props_sql}) " - f"AS SELECT * FROM {tmp_view}") + spark.sql("CREATE TABLE {} USING ICEBERG PARTITIONED BY (b) ".format(table) + + _NO_FANOUT + " AS SELECT * FROM {}".format(tmp_view)) with_cpu_session(setup_iceberg_table) assert with_gpu_session( diff --git a/jenkins/spark-tests.sh b/jenkins/spark-tests.sh index a1e04216de9..c2bd2d0da34 100755 --- a/jenkins/spark-tests.sh +++ b/jenkins/spark-tests.sh @@ -31,11 +31,41 @@ WORKSPACE=${WORKSPACE:-`pwd`} ARTF_ROOT="$WORKSPACE/jars" WGET_CMD="wget -q -P $ARTF_ROOT -t 3" PROJECT_REPO_HOST=$(sed -E 's#^(.*://)?([^/@]*@)?([^/:]+).*#\3#' <<< "$PROJECT_REPO") +CLASSIFIER=${CLASSIFIER:-"$CUDA_CLASSIFIER"} # default as CUDA_CLASSIFIER for compatibility rm -rf $ARTF_ROOT && mkdir -p $ARTF_ROOT -$WGET_CMD $PROJECT_TEST_REPO/com/nvidia/rapids-4-spark-integration-tests_$SCALA_BINARY_VER/$PROJECT_TEST_VER/rapids-4-spark-integration-tests_$SCALA_BINARY_VER-$PROJECT_TEST_VER-${SHUFFLE_SPARK_SHIM}.jar -CLASSIFIER=${CLASSIFIER:-"$CUDA_CLASSIFIER"} # default as CUDA_CLASSIFIER for compatibility +IT_BUILDVER=${SHUFFLE_SPARK_SHIM#spark} +IT_POM="$WORKSPACE/pom.xml" +IT_TARGET_DIR="$WORKSPACE/integration_tests/target" +if [[ "$SCALA_BINARY_VER" == "2.13" ]]; then + IT_POM="$WORKSPACE/scala2.13/pom.xml" + IT_TARGET_DIR="$WORKSPACE/scala2.13/integration_tests/target" +fi + +env -u SPARK_HOME $MVN -U -B ${MVN_URM_MIRROR:-} -f "$IT_POM" \ + -pl integration_tests clean package \ + -Dbuildver="$IT_BUILDVER" \ + -Dcuda.version="$CLASSIFIER" \ + -DskipTests \ + -Dmaven.scaladoc.skip \ + -Dmaven.scalastyle.skip=true \ + -Drat.skip=true + +RAPIDS_TEST_JAR=$(find "$IT_TARGET_DIR" -maxdepth 1 -type f \ + -name "rapids-4-spark-integration-tests_${SCALA_BINARY_VER}-*-${SHUFFLE_SPARK_SHIM}.jar" \ + -print -quit) +RAPIDS_INT_TESTS_TGZ=$(find "$IT_TARGET_DIR" -maxdepth 1 -type f \ + -name "rapids-4-spark-integration-tests_${SCALA_BINARY_VER}-*-${SHUFFLE_SPARK_SHIM}-pytest.tar.gz" \ + -print -quit) +if [[ -z "$RAPIDS_TEST_JAR" || -z "$RAPIDS_INT_TESTS_TGZ" ]]; then + echo "Failed to build integration-test artifacts from this branch" + exit 1 +fi +cp "$RAPIDS_TEST_JAR" "$RAPIDS_INT_TESTS_TGZ" "$ARTF_ROOT/" +RAPIDS_TEST_JAR="$ARTF_ROOT/$(basename "$RAPIDS_TEST_JAR")" +RAPIDS_INT_TESTS_TGZ="$ARTF_ROOT/$(basename "$RAPIDS_INT_TESTS_TGZ")" + if [ "$CLASSIFIER"x == x ];then $WGET_CMD $PROJECT_REPO/com/nvidia/rapids-4-spark_$SCALA_BINARY_VER/$PROJECT_VER/rapids-4-spark_$SCALA_BINARY_VER-${PROJECT_VER}.jar export RAPIDS_PLUGIN_JAR=$ARTF_ROOT/rapids-4-spark_${SCALA_BINARY_VER}-${PROJECT_VER}.jar @@ -43,18 +73,13 @@ else $WGET_CMD $PROJECT_REPO/com/nvidia/rapids-4-spark_$SCALA_BINARY_VER/$PROJECT_VER/rapids-4-spark_$SCALA_BINARY_VER-$PROJECT_VER-${CLASSIFIER}.jar export RAPIDS_PLUGIN_JAR="$ARTF_ROOT/rapids-4-spark_${SCALA_BINARY_VER}-$PROJECT_VER-${CLASSIFIER}.jar" fi -RAPIDS_TEST_JAR="$ARTF_ROOT/rapids-4-spark-integration-tests_${SCALA_BINARY_VER}-$PROJECT_TEST_VER-$SHUFFLE_SPARK_SHIM.jar" export INCLUDE_SPARK_AVRO_JAR=${INCLUDE_SPARK_AVRO_JAR:-"true"} if [[ "${INCLUDE_SPARK_AVRO_JAR}" == "true" ]]; then $WGET_CMD $SPARK_REPO/org/apache/spark/spark-avro_$SCALA_BINARY_VER/$SPARK_VER/spark-avro_$SCALA_BINARY_VER-${SPARK_VER}.jar fi -$WGET_CMD $PROJECT_TEST_REPO/com/nvidia/rapids-4-spark-integration-tests_$SCALA_BINARY_VER/$PROJECT_TEST_VER/rapids-4-spark-integration-tests_$SCALA_BINARY_VER-$PROJECT_TEST_VER-pytest.tar.gz - RAPIDS_INT_TESTS_HOME="$ARTF_ROOT/integration_tests/" -# The version of pytest.tar.gz that is uploaded is the one built against spark330 but its being pushed without classifier for now -RAPIDS_INT_TESTS_TGZ="$ARTF_ROOT/rapids-4-spark-integration-tests_${SCALA_BINARY_VER}-$PROJECT_TEST_VER-pytest.tar.gz" tmp_info=${TMP_INFO_FILE:-'/tmp/artifacts-build.info'} rm -rf "$tmp_info" @@ -86,10 +111,10 @@ set -x cat "$tmp_info" || true SKIP_REVISION_CHECK=${SKIP_REVISION_CHECK:-'false'} -if [[ "$SKIP_REVISION_CHECK" != "true" && (-z "$p_ver"|| \ - "$p_ver" != "$it_ver" || "$p_ver" != "$pt_ver") ]]; then - echo "Artifacts revisions are inconsistent!" - exit 1 +if [[ "$SKIP_REVISION_CHECK" != "true" && \ + (-z "$p_ver" || -z "$it_ver" || "$it_ver" != "$pt_ver") ]]; then + echo "Integration-test artifacts built from this branch are inconsistent!" + exit 1 fi tar xzf "$RAPIDS_INT_TESTS_TGZ" -C $ARTF_ROOT && rm -f "$RAPIDS_INT_TESTS_TGZ" From f49416787e7e3fc1093204bac33537dd80fbc27f Mon Sep 17 00:00:00 2001 From: Ray Liu Date: Wed, 29 Jul 2026 18:52:44 +0800 Subject: [PATCH 7/8] Fix test failure --- integration_tests/src/main/python/iceberg/iceberg_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/src/main/python/iceberg/iceberg_test.py b/integration_tests/src/main/python/iceberg/iceberg_test.py index d752266accd..108aeca2b81 100644 --- a/integration_tests/src/main/python/iceberg/iceberg_test.py +++ b/integration_tests/src/main/python/iceberg/iceberg_test.py @@ -648,8 +648,8 @@ def setup_iceberg_table(spark): with_cpu_session(setup_iceberg_table) assert with_gpu_session( - lambda spark: - spark._jvm.com.nvidia.spark.rapids.fileio.RapidsInputFiles.isS3PerfEnabled()), \ + lambda spark: spark.sparkContext.getConf().get( + 'spark.rapids.perfio.s3.enabled', 'false') == 'true'), \ "PerfIO S3 must be enabled at Spark startup for REST catalog tests" assert_gpu_and_cpu_are_equal_collect( lambda spark: spark.sql(f"SELECT * FROM {table}"), From 36fca111b8ab618b0befcac77707d59ba2a6a1a9 Mon Sep 17 00:00:00 2001 From: Ray Liu Date: Wed, 29 Jul 2026 19:35:18 +0800 Subject: [PATCH 8/8] Revert unnecessary chagnes Signed-off-by: Ray Liu --- jenkins/spark-tests.sh | 47 ++++++++++-------------------------------- 1 file changed, 11 insertions(+), 36 deletions(-) diff --git a/jenkins/spark-tests.sh b/jenkins/spark-tests.sh index c2bd2d0da34..a1e04216de9 100755 --- a/jenkins/spark-tests.sh +++ b/jenkins/spark-tests.sh @@ -31,41 +31,11 @@ WORKSPACE=${WORKSPACE:-`pwd`} ARTF_ROOT="$WORKSPACE/jars" WGET_CMD="wget -q -P $ARTF_ROOT -t 3" PROJECT_REPO_HOST=$(sed -E 's#^(.*://)?([^/@]*@)?([^/:]+).*#\3#' <<< "$PROJECT_REPO") -CLASSIFIER=${CLASSIFIER:-"$CUDA_CLASSIFIER"} # default as CUDA_CLASSIFIER for compatibility rm -rf $ARTF_ROOT && mkdir -p $ARTF_ROOT +$WGET_CMD $PROJECT_TEST_REPO/com/nvidia/rapids-4-spark-integration-tests_$SCALA_BINARY_VER/$PROJECT_TEST_VER/rapids-4-spark-integration-tests_$SCALA_BINARY_VER-$PROJECT_TEST_VER-${SHUFFLE_SPARK_SHIM}.jar -IT_BUILDVER=${SHUFFLE_SPARK_SHIM#spark} -IT_POM="$WORKSPACE/pom.xml" -IT_TARGET_DIR="$WORKSPACE/integration_tests/target" -if [[ "$SCALA_BINARY_VER" == "2.13" ]]; then - IT_POM="$WORKSPACE/scala2.13/pom.xml" - IT_TARGET_DIR="$WORKSPACE/scala2.13/integration_tests/target" -fi - -env -u SPARK_HOME $MVN -U -B ${MVN_URM_MIRROR:-} -f "$IT_POM" \ - -pl integration_tests clean package \ - -Dbuildver="$IT_BUILDVER" \ - -Dcuda.version="$CLASSIFIER" \ - -DskipTests \ - -Dmaven.scaladoc.skip \ - -Dmaven.scalastyle.skip=true \ - -Drat.skip=true - -RAPIDS_TEST_JAR=$(find "$IT_TARGET_DIR" -maxdepth 1 -type f \ - -name "rapids-4-spark-integration-tests_${SCALA_BINARY_VER}-*-${SHUFFLE_SPARK_SHIM}.jar" \ - -print -quit) -RAPIDS_INT_TESTS_TGZ=$(find "$IT_TARGET_DIR" -maxdepth 1 -type f \ - -name "rapids-4-spark-integration-tests_${SCALA_BINARY_VER}-*-${SHUFFLE_SPARK_SHIM}-pytest.tar.gz" \ - -print -quit) -if [[ -z "$RAPIDS_TEST_JAR" || -z "$RAPIDS_INT_TESTS_TGZ" ]]; then - echo "Failed to build integration-test artifacts from this branch" - exit 1 -fi -cp "$RAPIDS_TEST_JAR" "$RAPIDS_INT_TESTS_TGZ" "$ARTF_ROOT/" -RAPIDS_TEST_JAR="$ARTF_ROOT/$(basename "$RAPIDS_TEST_JAR")" -RAPIDS_INT_TESTS_TGZ="$ARTF_ROOT/$(basename "$RAPIDS_INT_TESTS_TGZ")" - +CLASSIFIER=${CLASSIFIER:-"$CUDA_CLASSIFIER"} # default as CUDA_CLASSIFIER for compatibility if [ "$CLASSIFIER"x == x ];then $WGET_CMD $PROJECT_REPO/com/nvidia/rapids-4-spark_$SCALA_BINARY_VER/$PROJECT_VER/rapids-4-spark_$SCALA_BINARY_VER-${PROJECT_VER}.jar export RAPIDS_PLUGIN_JAR=$ARTF_ROOT/rapids-4-spark_${SCALA_BINARY_VER}-${PROJECT_VER}.jar @@ -73,13 +43,18 @@ else $WGET_CMD $PROJECT_REPO/com/nvidia/rapids-4-spark_$SCALA_BINARY_VER/$PROJECT_VER/rapids-4-spark_$SCALA_BINARY_VER-$PROJECT_VER-${CLASSIFIER}.jar export RAPIDS_PLUGIN_JAR="$ARTF_ROOT/rapids-4-spark_${SCALA_BINARY_VER}-$PROJECT_VER-${CLASSIFIER}.jar" fi +RAPIDS_TEST_JAR="$ARTF_ROOT/rapids-4-spark-integration-tests_${SCALA_BINARY_VER}-$PROJECT_TEST_VER-$SHUFFLE_SPARK_SHIM.jar" export INCLUDE_SPARK_AVRO_JAR=${INCLUDE_SPARK_AVRO_JAR:-"true"} if [[ "${INCLUDE_SPARK_AVRO_JAR}" == "true" ]]; then $WGET_CMD $SPARK_REPO/org/apache/spark/spark-avro_$SCALA_BINARY_VER/$SPARK_VER/spark-avro_$SCALA_BINARY_VER-${SPARK_VER}.jar fi +$WGET_CMD $PROJECT_TEST_REPO/com/nvidia/rapids-4-spark-integration-tests_$SCALA_BINARY_VER/$PROJECT_TEST_VER/rapids-4-spark-integration-tests_$SCALA_BINARY_VER-$PROJECT_TEST_VER-pytest.tar.gz + RAPIDS_INT_TESTS_HOME="$ARTF_ROOT/integration_tests/" +# The version of pytest.tar.gz that is uploaded is the one built against spark330 but its being pushed without classifier for now +RAPIDS_INT_TESTS_TGZ="$ARTF_ROOT/rapids-4-spark-integration-tests_${SCALA_BINARY_VER}-$PROJECT_TEST_VER-pytest.tar.gz" tmp_info=${TMP_INFO_FILE:-'/tmp/artifacts-build.info'} rm -rf "$tmp_info" @@ -111,10 +86,10 @@ set -x cat "$tmp_info" || true SKIP_REVISION_CHECK=${SKIP_REVISION_CHECK:-'false'} -if [[ "$SKIP_REVISION_CHECK" != "true" && \ - (-z "$p_ver" || -z "$it_ver" || "$it_ver" != "$pt_ver") ]]; then - echo "Integration-test artifacts built from this branch are inconsistent!" - exit 1 +if [[ "$SKIP_REVISION_CHECK" != "true" && (-z "$p_ver"|| \ + "$p_ver" != "$it_ver" || "$p_ver" != "$pt_ver") ]]; then + echo "Artifacts revisions are inconsistent!" + exit 1 fi tar xzf "$RAPIDS_INT_TESTS_TGZ" -C $ARTF_ROOT && rm -f "$RAPIDS_INT_TESTS_TGZ"