diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index a8b2c6aa8d6f..3dca94e89965 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -963,7 +963,7 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ + $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi -Ppaimon -Pspark-ut \ -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest - name: Upload test report @@ -1401,6 +1401,7 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH java -version $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox \ + -Piceberg -Piceberg-test \ -Pspark-ut -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest - name: Upload test report diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/IcebergTestShims.java b/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/IcebergTestShims.java new file mode 100644 index 000000000000..2226767fdfac --- /dev/null +++ b/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/IcebergTestShims.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten; + +import org.apache.spark.rdd.RDD; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.types.StructType; + +public class IcebergTestShims { + + public static Dataset internalCreateDataFrame( + SparkSession spark, RDD rdd, StructType schema, boolean isStreaming) { + return spark.internalCreateDataFrame(rdd, schema, isStreaming); + } +} diff --git a/backends-velox/src-iceberg-spark35/test/java/org/apache/gluten/IcebergTestShims.java b/backends-velox/src-iceberg-spark35/test/java/org/apache/gluten/IcebergTestShims.java new file mode 100644 index 000000000000..2226767fdfac --- /dev/null +++ b/backends-velox/src-iceberg-spark35/test/java/org/apache/gluten/IcebergTestShims.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten; + +import org.apache.spark.rdd.RDD; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.types.StructType; + +public class IcebergTestShims { + + public static Dataset internalCreateDataFrame( + SparkSession spark, RDD rdd, StructType schema, boolean isStreaming) { + return spark.internalCreateDataFrame(rdd, schema, isStreaming); + } +} diff --git a/backends-velox/src-iceberg-spark40/test/java/org/apache/gluten/IcebergTestShims.java b/backends-velox/src-iceberg-spark40/test/java/org/apache/gluten/IcebergTestShims.java new file mode 100644 index 000000000000..92fec1985cb0 --- /dev/null +++ b/backends-velox/src-iceberg-spark40/test/java/org/apache/gluten/IcebergTestShims.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten; + +import org.apache.spark.rdd.RDD; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.types.StructType; + +public class IcebergTestShims { + + public static Dataset internalCreateDataFrame( + SparkSession spark, RDD rdd, StructType schema, boolean isStreaming) { + return ((org.apache.spark.sql.classic.SparkSession) spark) + .internalCreateDataFrame(rdd, schema, isStreaming); + } +} diff --git a/backends-velox/src-iceberg/main/java/org/apache/gluten/TestConfUtil.java b/backends-velox/src-iceberg/main/java/org/apache/gluten/TestConfUtil.java index 1ad6faca83e1..ec9e1d80da33 100644 --- a/backends-velox/src-iceberg/main/java/org/apache/gluten/TestConfUtil.java +++ b/backends-velox/src-iceberg/main/java/org/apache/gluten/TestConfUtil.java @@ -32,5 +32,7 @@ public class TestConfUtil { "spark.ui.enabled", "false", "spark.gluten.ui.enabled", + "false", + "spark.sql.ansi.enabled", "false"); } diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/execution/TestStoragePartitionedJoins.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/execution/TestStoragePartitionedJoins.java similarity index 99% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/execution/TestStoragePartitionedJoins.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/execution/TestStoragePartitionedJoins.java index 8123ca16fccd..6d3ac43e7894 100644 --- a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/execution/TestStoragePartitionedJoins.java +++ b/backends-velox/src-iceberg10/test/java/org/apache/gluten/execution/TestStoragePartitionedJoins.java @@ -16,6 +16,8 @@ */ package org.apache.gluten.execution; +import org.apache.gluten.IcebergTestShims; + import org.apache.commons.lang3.StringUtils; import org.apache.iceberg.PlanningMode; import org.apache.iceberg.Schema; @@ -655,7 +657,8 @@ private Dataset randomDataDF(Schema schema, int numRows) { Iterable rows = RandomData.generateSpark(schema, numRows, 0); JavaRDD rowRDD = sparkContext.parallelize(Lists.newArrayList(rows)); StructType rowSparkType = SparkSchemaUtil.convert(schema); - return spark.internalCreateDataFrame(JavaRDD.toRDD(rowRDD), rowSparkType, false); + return IcebergTestShims.internalCreateDataFrame( + spark, JavaRDD.toRDD(rowRDD), rowSparkType, false); } private void append(String table, Dataset df) throws NoSuchTableException { diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/execution/TestTPCHStoragePartitionedJoins.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/execution/TestTPCHStoragePartitionedJoins.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/execution/TestTPCHStoragePartitionedJoins.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/execution/TestTPCHStoragePartitionedJoins.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenCopyOnWriteDelete.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenCopyOnWriteDelete.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenCopyOnWriteDelete.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenCopyOnWriteDelete.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadDelete.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadDelete.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadDelete.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadDelete.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadMerge.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadMerge.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadMerge.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadMerge.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadUpdate.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadUpdate.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadUpdate.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadUpdate.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenStoragePartitionedJoinsInRowLevelOperations.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenStoragePartitionedJoinsInRowLevelOperations.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenStoragePartitionedJoinsInRowLevelOperations.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenStoragePartitionedJoinsInRowLevelOperations.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenSystemFunctionPushDownDQL.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenSystemFunctionPushDownDQL.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenSystemFunctionPushDownDQL.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenSystemFunctionPushDownDQL.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenSystemFunctionPushDownInRowLevelOperations.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenSystemFunctionPushDownInRowLevelOperations.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenSystemFunctionPushDownInRowLevelOperations.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/extensions/TestGlutenSystemFunctionPushDownInRowLevelOperations.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestFilteredScan.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestFilteredScan.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestFilteredScan.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestFilteredScan.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestForwardCompatibility.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestForwardCompatibility.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestForwardCompatibility.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestForwardCompatibility.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenDataFrameWriterV2.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenDataFrameWriterV2.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenDataFrameWriterV2.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenDataFrameWriterV2.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenDataFrameWriterV2Coercion.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenDataFrameWriterV2Coercion.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenDataFrameWriterV2Coercion.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenDataFrameWriterV2Coercion.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenDataSourceOptions.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenDataSourceOptions.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenDataSourceOptions.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenDataSourceOptions.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenIcebergSourceHiveTables.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenIcebergSourceHiveTables.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenIcebergSourceHiveTables.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenIcebergSourceHiveTables.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenIdentityPartitionData.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenIdentityPartitionData.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenIdentityPartitionData.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenIdentityPartitionData.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenPositionDeletesTable.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenPositionDeletesTable.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenPositionDeletesTable.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenPositionDeletesTable.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenRuntimeFiltering.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenRuntimeFiltering.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenRuntimeFiltering.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenRuntimeFiltering.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenSparkMetadataColumns.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenSparkMetadataColumns.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenSparkMetadataColumns.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenSparkMetadataColumns.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenSparkStagedScan.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenSparkStagedScan.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestGlutenSparkStagedScan.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestGlutenSparkStagedScan.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestIcebergSpark.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestIcebergSpark.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestIcebergSpark.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestIcebergSpark.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestParquetScan.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestParquetScan.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestParquetScan.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestParquetScan.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestPartitionPruning.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestPartitionPruning.java similarity index 99% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestPartitionPruning.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestPartitionPruning.java index ceb5799db7c0..58afffea169f 100644 --- a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestPartitionPruning.java +++ b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestPartitionPruning.java @@ -16,6 +16,7 @@ */ package org.apache.gluten.source; +import org.apache.gluten.IcebergTestShims; import org.apache.gluten.TestConfUtil; import org.apache.hadoop.conf.Configuration; @@ -331,8 +332,8 @@ private Dataset createTestDataset() { JavaRDD rdd = sparkContext.parallelize(rows); Dataset df = - spark.internalCreateDataFrame( - JavaRDD.toRDD(rdd), SparkSchemaUtil.convert(LOG_SCHEMA), false); + IcebergTestShims.internalCreateDataFrame( + spark, JavaRDD.toRDD(rdd), SparkSchemaUtil.convert(LOG_SCHEMA), false); return df.selectExpr("id", "date", "level", "message", "timestamp") .selectExpr( diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestPartitionValues.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestPartitionValues.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestPartitionValues.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestPartitionValues.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestSparkDataFile.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestSparkDataFile.java similarity index 98% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestSparkDataFile.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestSparkDataFile.java index 652cedd6aee9..725b73de645a 100644 --- a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestSparkDataFile.java +++ b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestSparkDataFile.java @@ -16,6 +16,7 @@ */ package org.apache.gluten.source; +import org.apache.gluten.IcebergTestShims; import org.apache.gluten.TestConfUtil; import org.apache.hadoop.conf.Configuration; @@ -139,8 +140,8 @@ private void checkSparkContentFiles(Table table) throws IOException { Iterable rows = RandomData.generateSpark(table.schema(), 200, 0); JavaRDD rdd = sparkContext.parallelize(Lists.newArrayList(rows)); Dataset df = - spark.internalCreateDataFrame( - JavaRDD.toRDD(rdd), SparkSchemaUtil.convert(table.schema()), false); + IcebergTestShims.internalCreateDataFrame( + spark, JavaRDD.toRDD(rdd), SparkSchemaUtil.convert(table.schema()), false); df.write().format("iceberg").mode("append").save(tableLocation); diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestSparkReaderWithBloomFilter.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestSparkReaderWithBloomFilter.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestSparkReaderWithBloomFilter.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestSparkReaderWithBloomFilter.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestTimestampWithoutZone.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestTimestampWithoutZone.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/source/TestTimestampWithoutZone.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/source/TestTimestampWithoutZone.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestFilterPushDown.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestFilterPushDown.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestFilterPushDown.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestFilterPushDown.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenAggregatePushDown.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenAggregatePushDown.java similarity index 98% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenAggregatePushDown.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenAggregatePushDown.java index 8433e9b2f425..b7442523b2af 100644 --- a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenAggregatePushDown.java +++ b/backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenAggregatePushDown.java @@ -36,7 +36,7 @@ public static void startMetastoreAndSpark() { TestBase.metastore = new TestHiveMetastore(); metastore.start(); TestBase.hiveConf = metastore.hiveConf(); - TestBase.spark.close(); + TestBase.spark.stop(); TestBase.spark = SparkSession.builder() .master("local[2]") diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenDeleteFrom.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenDeleteFrom.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenDeleteFrom.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenDeleteFrom.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesAsSelect.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesAsSelect.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesAsSelect.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesAsSelect.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesToBranch.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesToBranch.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesToBranch.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesToBranch.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesToWapBranch.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesToWapBranch.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesToWapBranch.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenPartitionedWritesToWapBranch.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenSelect.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenSelect.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenSelect.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenSelect.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenTimestampWithoutZone.java b/backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenTimestampWithoutZone.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/sql/TestGlutenTimestampWithoutZone.java rename to backends-velox/src-iceberg10/test/java/org/apache/gluten/sql/TestGlutenTimestampWithoutZone.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/CatalogTestBase.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/CatalogTestBase.java similarity index 90% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/CatalogTestBase.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/CatalogTestBase.java index 00b88080affa..5ae94b05aaf6 100644 --- a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/CatalogTestBase.java +++ b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/CatalogTestBase.java @@ -36,12 +36,7 @@ protected static Object[][] parameters() { SparkCatalogConfig.HADOOP.catalogName(), SparkCatalogConfig.HADOOP.implementation(), SparkCatalogConfig.HADOOP.properties() - }, - { - SparkCatalogConfig.SPARK.catalogName(), - SparkCatalogConfig.SPARK.implementation(), - SparkCatalogConfig.SPARK.properties() - }, + } }; } } diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java similarity index 92% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java index 6119d0df3b98..b8d1db25ca3a 100644 --- a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java +++ b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java @@ -39,11 +39,6 @@ public static Object[][] parameters() { SparkCatalogConfig.HADOOP.catalogName(), SparkCatalogConfig.HADOOP.implementation(), SparkCatalogConfig.HADOOP.properties() - }, - { - SparkCatalogConfig.SPARK.catalogName(), - SparkCatalogConfig.SPARK.implementation(), - SparkCatalogConfig.SPARK.properties() } }; } diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/SparkTestBase.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/SparkTestBase.java similarity index 98% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/SparkTestBase.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/SparkTestBase.java index 9d82ec7b4f93..53a07381a9c2 100644 --- a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/SparkTestBase.java +++ b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/SparkTestBase.java @@ -55,7 +55,7 @@ import java.util.concurrent.atomic.AtomicReference; import scala.Option; -import scala.collection.JavaConversions; +import scala.collection.JavaConverters; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTOREURIS; @@ -290,7 +290,7 @@ protected boolean checkAnswer(Dataset df) { () -> { Option msg = QueryTest.getErrorMessageInCheckAnswer( - df, JavaConversions.asScalaBuffer(rows).toSeq(), true); + df, JavaConverters.asScalaBuffer(rows).toSeq(), true); if (msg.isDefined()) { throw new RuntimeException(msg.get()); } diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/SparkTestBaseWithCatalog.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/SparkTestBaseWithCatalog.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/SparkTestBaseWithCatalog.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/SparkTestBaseWithCatalog.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/TestBase.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/TestBase.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/TestBase.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/TestBase.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/TestBaseWithCatalog.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/TestBaseWithCatalog.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/TestBaseWithCatalog.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/TestBaseWithCatalog.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/data/AvroDataTest.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/data/AvroDataTest.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/data/AvroDataTest.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/data/AvroDataTest.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/data/RandomData.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/data/RandomData.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/data/RandomData.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/data/RandomData.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/data/TestHelpers.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/data/TestHelpers.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/data/TestHelpers.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/data/TestHelpers.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java similarity index 99% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java index 9a750609fb51..34efbadd04a0 100644 --- a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java +++ b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java @@ -45,7 +45,7 @@ public static void startMetastoreAndSpark() { metastore.start(); TestBase.hiveConf = metastore.hiveConf(); - TestBase.spark.close(); + TestBase.spark.stop(); TestBase.spark = SparkSession.builder() diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/GlutenSparkScanBuilder.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/source/GlutenSparkScanBuilder.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/GlutenSparkScanBuilder.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/source/GlutenSparkScanBuilder.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/LogMessage.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/source/LogMessage.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/LogMessage.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/source/LogMessage.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java diff --git a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java similarity index 100% rename from backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java rename to backends-velox/src-iceberg10/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java diff --git a/pom.xml b/pom.xml index d84df1236d13..533efdd09ce1 100644 --- a/pom.xml +++ b/pom.xml @@ -73,6 +73,7 @@ 3.5.5 spark-sql-columnar-shims-spark35 1.10.0 + 10 1.2.0 delta-spark 3.3.2 @@ -1128,6 +1129,7 @@ spark-sql-columnar-shims-spark34 3.4.4 1.10.0 + 10 delta-core 2.4.0 24 @@ -1153,6 +1155,7 @@ spark-sql-columnar-shims-spark35 3.5.5 1.10.0 + 10 delta-spark 3.3.2 33 @@ -1190,7 +1193,9 @@ 4.0 spark-sql-columnar-shims-spark40 4.0.1 + 40 1.10.0 + 10 delta-spark 4.0.0 40 @@ -1622,6 +1627,8 @@ ${project.basedir}/src-iceberg/main/scala ${project.basedir}/src-iceberg/main/java + ${project.basedir}/src-iceberg${iceberg.binary.version}/main/scala + ${project.basedir}/src-iceberg${iceberg.binary.version}/main/java ${project.basedir}/src-iceberg-spark${spark.plain.version}/main/scala ${project.basedir}/src-iceberg-spark${spark.plain.version}/main/java @@ -1695,6 +1702,8 @@ generate-test-sources + ${project.basedir}/src-iceberg${iceberg.binary.version}/test/scala + ${project.basedir}/src-iceberg${iceberg.binary.version}/test/java ${project.basedir}/src-iceberg-spark${spark.plain.version}/test/scala ${project.basedir}/src-iceberg-spark${spark.plain.version}/test/java @@ -1709,6 +1718,7 @@ + ${project.basedir}/src-iceberg${iceberg.binary.version}/test/resources ${project.basedir}/src-iceberg-spark${spark.plain.version}/test/resources