From 9111136fa8e5536a6218de2e8079fbd70d37319c Mon Sep 17 00:00:00 2001 From: Chengcheng Jin Date: Tue, 2 Dec 2025 13:05:15 +0000 Subject: [PATCH 1/2] Spark35 and Spark40 add iceberg test --- .github/workflows/velox_backend_x86.yml | 3 +- .../org/apache/gluten/IcebergTestShims.java | 32 ++++++++++++++++++ .../org/apache/gluten/IcebergTestShims.java | 32 ++++++++++++++++++ .../org/apache/gluten/IcebergTestShims.java | 33 +++++++++++++++++++ .../java/org/apache/gluten/TestConfUtil.java | 4 ++- .../TestStoragePartitionedJoins.java | 5 ++- .../TestTPCHStoragePartitionedJoins.java | 0 .../TestGlutenCopyOnWriteDelete.java | 0 .../TestGlutenMergeOnReadDelete.java | 0 .../TestGlutenMergeOnReadMerge.java | 0 .../TestGlutenMergeOnReadUpdate.java | 0 ...ePartitionedJoinsInRowLevelOperations.java | 0 .../TestGlutenSystemFunctionPushDownDQL.java | 0 ...mFunctionPushDownInRowLevelOperations.java | 0 .../gluten/source/TestFilteredScan.java | 0 .../source/TestForwardCompatibility.java | 0 .../source/TestGlutenDataFrameWriterV2.java | 0 .../TestGlutenDataFrameWriterV2Coercion.java | 0 .../source/TestGlutenDataSourceOptions.java | 0 .../TestGlutenIcebergSourceHiveTables.java | 0 .../TestGlutenIdentityPartitionData.java | 0 .../TestGlutenPositionDeletesTable.java | 0 .../source/TestGlutenRuntimeFiltering.java | 0 .../TestGlutenSparkMetadataColumns.java | 0 .../source/TestGlutenSparkStagedScan.java | 0 .../gluten/source/TestIcebergSpark.java | 0 .../apache/gluten/source/TestParquetScan.java | 0 .../gluten/source/TestPartitionPruning.java | 5 +-- .../gluten/source/TestPartitionValues.java | 0 .../gluten/source/TestSparkDataFile.java | 5 +-- .../TestSparkReaderWithBloomFilter.java | 0 .../source/TestTimestampWithoutZone.java | 0 .../apache/gluten/sql/TestFilterPushDown.java | 0 .../sql/TestGlutenAggregatePushDown.java | 2 +- .../gluten/sql/TestGlutenDeleteFrom.java | 0 .../TestGlutenPartitionedWritesAsSelect.java | 0 .../TestGlutenPartitionedWritesToBranch.java | 0 ...estGlutenPartitionedWritesToWapBranch.java | 0 .../apache/gluten/sql/TestGlutenSelect.java | 0 .../sql/TestGlutenTimestampWithoutZone.java | 0 .../apache/iceberg/spark/CatalogTestBase.java | 7 +--- .../iceberg/spark/SparkCatalogTestBase.java | 5 --- .../apache/iceberg/spark/SparkTestBase.java | 4 +-- .../spark/SparkTestBaseWithCatalog.java | 0 .../org/apache/iceberg/spark/TestBase.java | 0 .../iceberg/spark/TestBaseWithCatalog.java | 0 .../iceberg/spark/data/AvroDataTest.java | 0 .../apache/iceberg/spark/data/RandomData.java | 0 .../iceberg/spark/data/TestHelpers.java | 0 .../spark/extensions/ExtensionsTestBase.java | 2 +- .../extensions/SparkExtensionsTestBase.java | 0 .../spark/source/GlutenSparkScanBuilder.java | 0 .../iceberg/spark/source/LogMessage.java | 0 .../source/TestIcebergSourceTablesBase.java | 0 .../spark/source/TestSparkReadProjection.java | 0 .../spark/source/TestSparkReaderDeletes.java | 0 .../spark/source/TestStructuredStreaming.java | 0 pom.xml | 10 ++++++ 58 files changed, 127 insertions(+), 22 deletions(-) create mode 100644 backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/IcebergTestShims.java create mode 100644 backends-velox/src-iceberg-spark35/test/java/org/apache/gluten/IcebergTestShims.java create mode 100644 backends-velox/src-iceberg-spark40/test/java/org/apache/gluten/IcebergTestShims.java rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/execution/TestStoragePartitionedJoins.java (99%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/execution/TestTPCHStoragePartitionedJoins.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/extensions/TestGlutenCopyOnWriteDelete.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadDelete.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadMerge.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadUpdate.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/extensions/TestGlutenStoragePartitionedJoinsInRowLevelOperations.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/extensions/TestGlutenSystemFunctionPushDownDQL.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/extensions/TestGlutenSystemFunctionPushDownInRowLevelOperations.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/source/TestFilteredScan.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/source/TestForwardCompatibility.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/source/TestGlutenDataFrameWriterV2.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/source/TestGlutenDataFrameWriterV2Coercion.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/source/TestGlutenDataSourceOptions.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/source/TestGlutenIcebergSourceHiveTables.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/source/TestGlutenIdentityPartitionData.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/source/TestGlutenPositionDeletesTable.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/source/TestGlutenRuntimeFiltering.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/source/TestGlutenSparkMetadataColumns.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/source/TestGlutenSparkStagedScan.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/source/TestIcebergSpark.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/source/TestParquetScan.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/source/TestPartitionPruning.java (99%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/source/TestPartitionValues.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/source/TestSparkDataFile.java (98%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/source/TestSparkReaderWithBloomFilter.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/source/TestTimestampWithoutZone.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/sql/TestFilterPushDown.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/sql/TestGlutenAggregatePushDown.java (98%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/sql/TestGlutenDeleteFrom.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesAsSelect.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesToBranch.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesToWapBranch.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/sql/TestGlutenSelect.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/gluten/sql/TestGlutenTimestampWithoutZone.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/iceberg/spark/CatalogTestBase.java (90%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java (92%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/iceberg/spark/SparkTestBase.java (98%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/iceberg/spark/SparkTestBaseWithCatalog.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/iceberg/spark/TestBase.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/iceberg/spark/TestBaseWithCatalog.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/iceberg/spark/data/AvroDataTest.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/iceberg/spark/data/RandomData.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/iceberg/spark/data/TestHelpers.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java (99%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/iceberg/spark/source/GlutenSparkScanBuilder.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/iceberg/spark/source/LogMessage.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java (100%) rename backends-velox/{src-iceberg-spark34 => src-iceberg10}/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java (100%) diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index a8b2c6aa8d6f..3dca94e89965 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -963,7 +963,7 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ + $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi -Ppaimon -Pspark-ut \ -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest - name: Upload test report @@ -1401,6 +1401,7 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH java -version $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox \ + -Piceberg -Piceberg-test \ -Pspark-ut -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest - name: Upload test report diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/IcebergTestShims.java b/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/IcebergTestShims.java new file mode 100644 index 000000000000..2226767fdfac --- /dev/null +++ b/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/IcebergTestShims.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten; + +import org.apache.spark.rdd.RDD; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.types.StructType; + +public class IcebergTestShims { + + public static Dataset internalCreateDataFrame( + SparkSession spark, RDD rdd, StructType schema, boolean isStreaming) { + return spark.internalCreateDataFrame(rdd, schema, isStreaming); + } +} diff --git a/backends-velox/src-iceberg-spark35/test/java/org/apache/gluten/IcebergTestShims.java b/backends-velox/src-iceberg-spark35/test/java/org/apache/gluten/IcebergTestShims.java new file mode 100644 index 000000000000..2226767fdfac --- /dev/null +++ b/backends-velox/src-iceberg-spark35/test/java/org/apache/gluten/IcebergTestShims.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten; + +import org.apache.spark.rdd.RDD; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.types.StructType; + +public class IcebergTestShims { + + public static Dataset internalCreateDataFrame( + SparkSession spark, RDD rdd, StructType schema, boolean isStreaming) { + return spark.internalCreateDataFrame(rdd, schema, isStreaming); + } +} diff --git a/backends-velox/src-iceberg-spark40/test/java/org/apache/gluten/IcebergTestShims.java b/backends-velox/src-iceberg-spark40/test/java/org/apache/gluten/IcebergTestShims.java new file mode 100644 index 000000000000..92fec1985cb0 --- /dev/null +++ b/backends-velox/src-iceberg-spark40/test/java/org/apache/gluten/IcebergTestShims.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten; + +import org.apache.spark.rdd.RDD; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.types.StructType; + +public class IcebergTestShims { + + public static Dataset internalCreateDataFrame( + SparkSession spark, RDD rdd, StructType schema, boolean isStreaming) { + return ((org.apache.spark.sql.classic.SparkSession) spark) + .internalCreateDataFrame(rdd, schema, isStreaming); + } +} diff --git a/backends-velox/src-iceberg/main/java/org/apache/gluten/TestConfUtil.java b/backends-velox/src-iceberg/main/java/org/apache/gluten/TestConfUtil.java index 1ad6faca83e1..ba9204ceaa87 100644 --- a/backends-velox/src-iceberg/main/java/org/apache/gluten/TestConfUtil.java +++ b/backends-velox/src-iceberg/main/java/org/apache/gluten/TestConfUtil.java @@ -32,5 +32,7 @@ public class TestConfUtil { "spark.ui.enabled", "false", "spark.gluten.ui.enabled", - "false"); + "false", + "spark.sql.ansi.enabled", + "false"); } diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/execution/TestStoragePartitionedJoins.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/execution/TestStoragePartitionedJoins.java similarity index 99% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/execution/TestStoragePartitionedJoins.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/execution/TestStoragePartitionedJoins.java index 8123ca16fccd..6d3ac43e7894 100644 --- a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/execution/TestStoragePartitionedJoins.java +++ b/backends-velox/src-iceberg10/test/java/org/apache/gluten/execution/TestStoragePartitionedJoins.java @@ -16,6 +16,8 @@ */ package org.apache.gluten.execution; +import org.apache.gluten.IcebergTestShims; + import org.apache.commons.lang3.StringUtils; import org.apache.iceberg.PlanningMode; import org.apache.iceberg.Schema; @@ -655,7 +657,8 @@ private Dataset randomDataDF(Schema schema, int numRows) { Iterable rows = RandomData.generateSpark(schema, numRows, 0); JavaRDD rowRDD = sparkContext.parallelize(Lists.newArrayList(rows)); StructType rowSparkType = SparkSchemaUtil.convert(schema); - return spark.internalCreateDataFrame(JavaRDD.toRDD(rowRDD), rowSparkType, false); + return IcebergTestShims.internalCreateDataFrame( + spark, JavaRDD.toRDD(rowRDD), rowSparkType, false); } private void append(String table, Dataset df) throws NoSuchTableException { diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/execution/TestTPCHStoragePartitionedJoins.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/execution/TestTPCHStoragePartitionedJoins.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/execution/TestTPCHStoragePartitionedJoins.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/execution/TestTPCHStoragePartitionedJoins.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenCopyOnWriteDelete.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenCopyOnWriteDelete.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenCopyOnWriteDelete.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenCopyOnWriteDelete.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadDelete.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadDelete.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadDelete.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadDelete.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadMerge.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadMerge.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadMerge.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadMerge.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadUpdate.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadUpdate.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadUpdate.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadUpdate.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenStoragePartitionedJoinsInRowLevelOperations.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenStoragePartitionedJoinsInRowLevelOperations.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenStoragePartitionedJoinsInRowLevelOperations.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenStoragePartitionedJoinsInRowLevelOperations.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenSystemFunctionPushDownDQL.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenSystemFunctionPushDownDQL.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenSystemFunctionPushDownDQL.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenSystemFunctionPushDownDQL.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenSystemFunctionPushDownInRowLevelOperations.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenSystemFunctionPushDownInRowLevelOperations.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenSystemFunctionPushDownInRowLevelOperations.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenSystemFunctionPushDownInRowLevelOperations.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestFilteredScan.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestFilteredScan.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestFilteredScan.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestFilteredScan.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestForwardCompatibility.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestForwardCompatibility.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestForwardCompatibility.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestForwardCompatibility.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenDataFrameWriterV2.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenDataFrameWriterV2.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenDataFrameWriterV2.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenDataFrameWriterV2.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenDataFrameWriterV2Coercion.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenDataFrameWriterV2Coercion.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenDataFrameWriterV2Coercion.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenDataFrameWriterV2Coercion.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenDataSourceOptions.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenDataSourceOptions.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenDataSourceOptions.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenDataSourceOptions.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenIcebergSourceHiveTables.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenIcebergSourceHiveTables.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenIcebergSourceHiveTables.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenIcebergSourceHiveTables.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenIdentityPartitionData.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenIdentityPartitionData.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenIdentityPartitionData.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenIdentityPartitionData.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenPositionDeletesTable.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenPositionDeletesTable.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenPositionDeletesTable.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenPositionDeletesTable.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenRuntimeFiltering.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenRuntimeFiltering.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenRuntimeFiltering.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenRuntimeFiltering.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenSparkMetadataColumns.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenSparkMetadataColumns.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenSparkMetadataColumns.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenSparkMetadataColumns.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenSparkStagedScan.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenSparkStagedScan.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenSparkStagedScan.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenSparkStagedScan.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestIcebergSpark.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestIcebergSpark.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestIcebergSpark.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestIcebergSpark.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestParquetScan.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestParquetScan.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestParquetScan.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestParquetScan.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestPartitionPruning.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestPartitionPruning.java similarity index 99% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestPartitionPruning.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestPartitionPruning.java index ceb5799db7c0..58afffea169f 100644 --- a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestPartitionPruning.java +++ b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestPartitionPruning.java @@ -16,6 +16,7 @@ */ package org.apache.gluten.source; +import org.apache.gluten.IcebergTestShims; import org.apache.gluten.TestConfUtil; import org.apache.hadoop.conf.Configuration; @@ -331,8 +332,8 @@ private Dataset createTestDataset() { JavaRDD rdd = sparkContext.parallelize(rows); Dataset df = - spark.internalCreateDataFrame( - JavaRDD.toRDD(rdd), SparkSchemaUtil.convert(LOG_SCHEMA), false); + IcebergTestShims.internalCreateDataFrame( + spark, JavaRDD.toRDD(rdd), SparkSchemaUtil.convert(LOG_SCHEMA), false); return df.selectExpr("id", "date", "level", "message", "timestamp") .selectExpr( diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestPartitionValues.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestPartitionValues.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestPartitionValues.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestPartitionValues.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestSparkDataFile.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestSparkDataFile.java similarity index 98% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestSparkDataFile.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestSparkDataFile.java index 652cedd6aee9..725b73de645a 100644 --- a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestSparkDataFile.java +++ b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestSparkDataFile.java @@ -16,6 +16,7 @@ */ package org.apache.gluten.source; +import org.apache.gluten.IcebergTestShims; import org.apache.gluten.TestConfUtil; import org.apache.hadoop.conf.Configuration; @@ -139,8 +140,8 @@ private void checkSparkContentFiles(Table table) throws IOException { Iterable rows = RandomData.generateSpark(table.schema(), 200, 0); JavaRDD rdd = sparkContext.parallelize(Lists.newArrayList(rows)); Dataset df = - spark.internalCreateDataFrame( - JavaRDD.toRDD(rdd), SparkSchemaUtil.convert(table.schema()), false); + IcebergTestShims.internalCreateDataFrame( + spark, JavaRDD.toRDD(rdd), SparkSchemaUtil.convert(table.schema()), false); df.write().format("iceberg").mode("append").save(tableLocation); diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestSparkReaderWithBloomFilter.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestSparkReaderWithBloomFilter.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestSparkReaderWithBloomFilter.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestSparkReaderWithBloomFilter.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestTimestampWithoutZone.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestTimestampWithoutZone.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestTimestampWithoutZone.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestTimestampWithoutZone.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestFilterPushDown.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestFilterPushDown.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestFilterPushDown.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestFilterPushDown.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenAggregatePushDown.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenAggregatePushDown.java similarity index 98% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenAggregatePushDown.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenAggregatePushDown.java index 8433e9b2f425..b7442523b2af 100644 --- a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenAggregatePushDown.java +++ b/backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenAggregatePushDown.java @@ -36,7 +36,7 @@ public static void startMetastoreAndSpark() { TestBase.metastore = new TestHiveMetastore(); metastore.start(); TestBase.hiveConf = metastore.hiveConf(); - TestBase.spark.close(); + TestBase.spark.stop(); TestBase.spark = SparkSession.builder() .master("local[2]") diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenDeleteFrom.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenDeleteFrom.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenDeleteFrom.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenDeleteFrom.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesAsSelect.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesAsSelect.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesAsSelect.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesAsSelect.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesToBranch.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesToBranch.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesToBranch.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesToBranch.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesToWapBranch.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesToWapBranch.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesToWapBranch.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesToWapBranch.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenSelect.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenSelect.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenSelect.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenSelect.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenTimestampWithoutZone.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenTimestampWithoutZone.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenTimestampWithoutZone.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenTimestampWithoutZone.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/CatalogTestBase.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/CatalogTestBase.java similarity index 90% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/CatalogTestBase.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/CatalogTestBase.java index 00b88080affa..5ae94b05aaf6 100644 --- a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/CatalogTestBase.java +++ b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/CatalogTestBase.java @@ -36,12 +36,7 @@ protected static Object[][] parameters() { SparkCatalogConfig.HADOOP.catalogName(), SparkCatalogConfig.HADOOP.implementation(), SparkCatalogConfig.HADOOP.properties() - }, - { - SparkCatalogConfig.SPARK.catalogName(), - SparkCatalogConfig.SPARK.implementation(), - SparkCatalogConfig.SPARK.properties() - }, + } }; } } diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java similarity index 92% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java index 6119d0df3b98..b8d1db25ca3a 100644 --- a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java +++ b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java @@ -39,11 +39,6 @@ public static Object[][] parameters() { SparkCatalogConfig.HADOOP.catalogName(), SparkCatalogConfig.HADOOP.implementation(), SparkCatalogConfig.HADOOP.properties() - }, - { - SparkCatalogConfig.SPARK.catalogName(), - SparkCatalogConfig.SPARK.implementation(), - SparkCatalogConfig.SPARK.properties() } }; } diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/SparkTestBase.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/SparkTestBase.java similarity index 98% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/SparkTestBase.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/SparkTestBase.java index 9d82ec7b4f93..53a07381a9c2 100644 --- a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/SparkTestBase.java +++ b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/SparkTestBase.java @@ -55,7 +55,7 @@ import java.util.concurrent.atomic.AtomicReference; import scala.Option; -import scala.collection.JavaConversions; +import scala.collection.JavaConverters; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTOREURIS; @@ -290,7 +290,7 @@ protected boolean checkAnswer(Dataset df) { () -> { Option msg = QueryTest.getErrorMessageInCheckAnswer( - df, JavaConversions.asScalaBuffer(rows).toSeq(), true); + df, JavaConverters.asScalaBuffer(rows).toSeq(), true); if (msg.isDefined()) { throw new RuntimeException(msg.get()); } diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/SparkTestBaseWithCatalog.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/SparkTestBaseWithCatalog.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/SparkTestBaseWithCatalog.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/SparkTestBaseWithCatalog.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/TestBase.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/TestBase.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/TestBase.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/TestBase.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/TestBaseWithCatalog.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/TestBaseWithCatalog.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/TestBaseWithCatalog.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/TestBaseWithCatalog.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/data/AvroDataTest.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/data/AvroDataTest.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/data/AvroDataTest.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/data/AvroDataTest.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/data/RandomData.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/data/RandomData.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/data/RandomData.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/data/RandomData.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/data/TestHelpers.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/data/TestHelpers.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/data/TestHelpers.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/data/TestHelpers.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java similarity index 99% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java index 9a750609fb51..34efbadd04a0 100644 --- a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java +++ b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java @@ -45,7 +45,7 @@ public static void startMetastoreAndSpark() { metastore.start(); TestBase.hiveConf = metastore.hiveConf(); - TestBase.spark.close(); + TestBase.spark.stop(); TestBase.spark = SparkSession.builder() diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/GlutenSparkScanBuilder.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/source/GlutenSparkScanBuilder.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/GlutenSparkScanBuilder.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/source/GlutenSparkScanBuilder.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/LogMessage.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/source/LogMessage.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/LogMessage.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/source/LogMessage.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java diff --git a/pom.xml b/pom.xml index d84df1236d13..533efdd09ce1 100644 --- a/pom.xml +++ b/pom.xml @@ -73,6 +73,7 @@ 3.5.5 spark-sql-columnar-shims-spark35 1.10.0 + 10 1.2.0 delta-spark 3.3.2 @@ -1128,6 +1129,7 @@ spark-sql-columnar-shims-spark34 3.4.4 1.10.0 + 10 delta-core 2.4.0 24 @@ -1153,6 +1155,7 @@ spark-sql-columnar-shims-spark35 3.5.5 1.10.0 + 10 delta-spark 3.3.2 33 @@ -1190,7 +1193,9 @@ 4.0 spark-sql-columnar-shims-spark40 4.0.1 + 40 1.10.0 + 10 delta-spark 4.0.0 40 @@ -1622,6 +1627,8 @@ ${project.basedir}/src-iceberg/main/scala ${project.basedir}/src-iceberg/main/java + ${project.basedir}/src-iceberg${iceberg.binary.version}/main/scala + ${project.basedir}/src-iceberg${iceberg.binary.version}/main/java ${project.basedir}/src-iceberg-spark${spark.plain.version}/main/scala ${project.basedir}/src-iceberg-spark${spark.plain.version}/main/java @@ -1695,6 +1702,8 @@ generate-test-sources + ${project.basedir}/src-iceberg${iceberg.binary.version}/test/scala + ${project.basedir}/src-iceberg${iceberg.binary.version}/test/java ${project.basedir}/src-iceberg-spark${spark.plain.version}/test/scala ${project.basedir}/src-iceberg-spark${spark.plain.version}/test/java @@ -1709,6 +1718,7 @@ + ${project.basedir}/src-iceberg${iceberg.binary.version}/test/resources ${project.basedir}/src-iceberg-spark${spark.plain.version}/test/resources From edc4709a375be977dfc6417c0fe7712edc89a60b Mon Sep 17 00:00:00 2001 From: Chengcheng Jin Date: Tue, 2 Dec 2025 15:13:29 +0000 Subject: [PATCH 2/2] fix code style --- .../src-iceberg/main/java/org/apache/gluten/TestConfUtil.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backends-velox/src-iceberg/main/java/org/apache/gluten/TestConfUtil.java b/backends-velox/src-iceberg/main/java/org/apache/gluten/TestConfUtil.java index ba9204ceaa87..ec9e1d80da33 100644 --- a/backends-velox/src-iceberg/main/java/org/apache/gluten/TestConfUtil.java +++ b/backends-velox/src-iceberg/main/java/org/apache/gluten/TestConfUtil.java @@ -33,6 +33,6 @@ public class TestConfUtil { "false", "spark.gluten.ui.enabled", "false", - "spark.sql.ansi.enabled", - "false"); + "spark.sql.ansi.enabled", + "false"); }