diff --git a/aggregator/pom.xml b/aggregator/pom.xml index 7b48ff2afb7..5f1ecd585b6 100644 --- a/aggregator/pom.xml +++ b/aggregator/pom.xml @@ -762,5 +762,22 @@ + + release350emr + + + buildver + 350emr + + + + + com.nvidia + rapids-4-spark-delta-stub_${scala.binary.version} + ${project.version} + ${spark.version.classifier} + + + diff --git a/datagen/src/main/spark320/scala/org/apache/spark/sql/tests/datagen/datagen/DataGenExprBase.scala b/datagen/src/main/spark320/scala/org/apache/spark/sql/tests/datagen/datagen/DataGenExprBase.scala index 1d3cad311b5..7e7f44ba494 100644 --- a/datagen/src/main/spark320/scala/org/apache/spark/sql/tests/datagen/datagen/DataGenExprBase.scala +++ b/datagen/src/main/spark320/scala/org/apache/spark/sql/tests/datagen/datagen/DataGenExprBase.scala @@ -36,6 +36,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.tests.datagen diff --git a/integration_tests/src/main/python/arithmetic_ops_test.py b/integration_tests/src/main/python/arithmetic_ops_test.py index b75872ed8b2..57caeea4a86 100644 --- a/integration_tests/src/main/python/arithmetic_ops_test.py +++ b/integration_tests/src/main/python/arithmetic_ops_test.py @@ -160,6 +160,8 @@ def test_subtraction_ansi_no_overflow(data_gen): _decimal_gen_38_10, _decimal_gen_38_neg10 ], ids=idfn) +@pytest.mark.xfail(condition=is_spark_350emr(), + reason='EMR back-ported https://issues.apache.org/jira/browse/SPARK-45786 in spark350emr') def test_multiplication(data_gen): data_type = data_gen.data_type assert_gpu_and_cpu_are_equal_collect( @@ -203,6 +205,8 @@ def test_multiplication_ansi_overflow(): @pytest.mark.parametrize('rhs', [byte_gen, short_gen, int_gen, long_gen, DecimalGen(6, 3), DecimalGen(10, -2), DecimalGen(15, 3), DecimalGen(30, 12), DecimalGen(3, -3), DecimalGen(27, 7), DecimalGen(20, -3)], ids=idfn) +@pytest.mark.xfail(condition=is_spark_350emr(), + reason='EMR back-ported https://issues.apache.org/jira/browse/SPARK-45786 in spark350emr') def test_multiplication_mixed(lhs, rhs): assert_gpu_and_cpu_are_equal_collect( lambda spark : two_col_df(spark, lhs, rhs).select( diff --git a/integration_tests/src/main/python/spark_session.py b/integration_tests/src/main/python/spark_session.py index 17ba470f168..b8863f2c255 100644 --- a/integration_tests/src/main/python/spark_session.py +++ b/integration_tests/src/main/python/spark_session.py @@ -241,6 +241,12 @@ def is_spark_332cdh(): def is_spark_cdh(): return is_spark_321cdh() or is_spark_330cdh() or is_spark_332cdh() +def is_spark_emr(): + return is_spark350emr() + +def is_spark_350emr(): + return "3.5.0-amzn" in spark_version() + def is_databricks_version_or_later(major, minor): spark = get_spark_i_know_what_i_am_doing() version = spark.conf.get("spark.databricks.clusterUsageTags.sparkVersion", "0.0") diff --git a/pom.xml b/pom.xml index fd1c821cfc2..0067fd46346 100644 --- a/pom.xml +++ b/pom.xml @@ -595,6 +595,27 @@ delta-lake/delta-stub + + release350emr + + + buildver + 350emr + + + + 350emr + ${spark350emr.version} + ${spark350emr.version} + 1.13.1 + ${spark330.iceberg.version} + 2.0.7 + + + shim-deps/emr + delta-lake/delta-stub + + release351 @@ -782,6 +803,7 @@ 3.3.2-databricks 3.4.1-databricks 3.5.0 + 3.5.0-amzn-0 3.5.1-SNAPSHOT 3.12.4 4.3.0 @@ -791,6 +813,7 @@ 30.0-jre 2.0.0 1.7.30 + 2.20.0 1.11.0 3.3.1 0.13.2 @@ -831,7 +854,8 @@ 340, 341, 342, - 350 + 350, + 350emr 351 @@ -933,6 +957,34 @@ jucx ${ucx.version} + + + org.slf4j + slf4j-api + ${slf4j.version} + + + org.apache.logging.log4j + log4j-slf4j2-impl + ${log4j.version} + + + org.apache.logging.log4j + log4j-api + ${log4j.version} + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + + + + org.apache.logging.log4j + log4j-1.2-api + ${log4j.version} + + org.slf4j jul-to-slf4j diff --git a/scala2.13/aggregator/pom.xml b/scala2.13/aggregator/pom.xml index 9d8258943ce..83fac60982e 100644 --- a/scala2.13/aggregator/pom.xml +++ b/scala2.13/aggregator/pom.xml @@ -762,5 +762,22 @@ + + release350emr + + + buildver + 350emr + + + + + com.nvidia + rapids-4-spark-delta-stub_${scala.binary.version} + ${project.version} + ${spark.version.classifier} + + + diff --git a/scala2.13/pom.xml b/scala2.13/pom.xml index 5ccc17c4b70..6e2e270df7e 100644 --- a/scala2.13/pom.xml +++ b/scala2.13/pom.xml @@ -595,6 +595,27 @@ delta-lake/delta-stub + + release350emr + + + buildver + 350emr + + + + 350emr + ${spark350emr.version} + ${spark350emr.version} + 1.13.1 + ${spark330.iceberg.version} + 2.0.7 + + + shim-deps/emr + delta-lake/delta-stub + + release351 @@ -782,6 +803,7 @@ 3.3.2-databricks 3.4.1-databricks 3.5.0 + 3.5.0-amzn-0 3.5.1-SNAPSHOT 3.12.4 4.3.0 diff --git a/scala2.13/shim-deps/emr/pom.xml b/scala2.13/shim-deps/emr/pom.xml new file mode 100644 index 00000000000..65380831f4b --- /dev/null +++ b/scala2.13/shim-deps/emr/pom.xml @@ -0,0 +1,59 @@ + + + + 4.0.0 + + + com.nvidia + rapids-4-spark-parent_2.12 + 24.02.0-SNAPSHOT + ../../pom.xml + + rapids-4-spark-emr-bom + pom + EMR Shim Dependencies + 24.02.0-SNAPSHOT + + + ../shim-deps/emr + + + + + + org.apache.spark + spark-sql_${scala.binary.version} + compile + + + org.apache.spark + spark-hive_${scala.binary.version} + compile + + + org.apache.arrow + * + + + + + \ No newline at end of file diff --git a/scala2.13/shim-deps/pom.xml b/scala2.13/shim-deps/pom.xml index 8265e726ee0..91524a7669f 100644 --- a/scala2.13/shim-deps/pom.xml +++ b/scala2.13/shim-deps/pom.xml @@ -176,6 +176,24 @@ + + release350emr + + + buildver + 350emr + + + + + com.nvidia + rapids-4-spark-emr-bom + ${project.version} + pom + provided + + + with-classifier diff --git a/shim-deps/emr/pom.xml b/shim-deps/emr/pom.xml new file mode 100644 index 00000000000..6ceedef9745 --- /dev/null +++ b/shim-deps/emr/pom.xml @@ -0,0 +1,59 @@ + + + + 4.0.0 + + + com.nvidia + rapids-4-spark-parent_2.12 + 24.02.0-SNAPSHOT + ../../pom.xml + + rapids-4-spark-emr-bom + pom + EMR Shim Dependencies + 24.02.0-SNAPSHOT + + + ../shim-deps/emr + + + + + + org.apache.spark + spark-sql_${scala.binary.version} + compile + + + org.apache.spark + spark-hive_${scala.binary.version} + compile + + + org.apache.arrow + * + + + + + \ No newline at end of file diff --git a/shim-deps/pom.xml b/shim-deps/pom.xml index 0d63398c1c8..73867d6bd64 100644 --- a/shim-deps/pom.xml +++ b/shim-deps/pom.xml @@ -176,6 +176,24 @@ + + release350emr + + + buildver + 350emr + + + + + com.nvidia + rapids-4-spark-emr-bom + ${project.version} + pom + provided + + + with-classifier diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AQEUtils.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AQEUtils.scala index 864b9175d4e..720375b98f6 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AQEUtils.scala +++ b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AQEUtils.scala @@ -35,6 +35,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AggregationTagging.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AggregationTagging.scala index e97126ba478..3bd19e43eda 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AggregationTagging.scala +++ b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AggregationTagging.scala @@ -35,6 +35,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DecimalMultiply128.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DecimalMultiply128.scala index 014d6ba474a..b25d45524dd 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DecimalMultiply128.scala +++ b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DecimalMultiply128.scala @@ -38,6 +38,7 @@ {"spark": "341"} {"spark": "341db"} {"spark": "350"} +{"spark": "350emr"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DeltaLakeUtils.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DeltaLakeUtils.scala index 4dc843f3ef3..502c0b4514d 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DeltaLakeUtils.scala +++ b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DeltaLakeUtils.scala @@ -35,6 +35,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GetSequenceSize.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GetSequenceSize.scala index e906a94df13..2c00f9a3162 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GetSequenceSize.scala +++ b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GetSequenceSize.scala @@ -37,6 +37,7 @@ {"spark": "341"} {"spark": "341db"} {"spark": "350"} +{"spark": "350emr"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuFileFormatDataWriterShim.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuFileFormatDataWriterShim.scala index bb27199b6c2..2409ff40535 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuFileFormatDataWriterShim.scala +++ b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuFileFormatDataWriterShim.scala @@ -37,6 +37,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuWindowInPandasExec.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuWindowInPandasExec.scala index f1b2193a5ff..f439a475655 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuWindowInPandasExec.scala +++ b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuWindowInPandasExec.scala @@ -35,6 +35,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ReaderUtils.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ReaderUtils.scala index 05b8fea887f..bbcc237a966 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ReaderUtils.scala +++ b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ReaderUtils.scala @@ -36,6 +36,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimLeafExecNode.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimLeafExecNode.scala index cca77aec318..8ebb5166c28 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimLeafExecNode.scala +++ b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimLeafExecNode.scala @@ -35,6 +35,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/extractValueShims.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/extractValueShims.scala index 38a2bc82557..fbdca1ca7cd 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/extractValueShims.scala +++ b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/extractValueShims.scala @@ -36,6 +36,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/rapids/shims/GpuShuffleExchangeExec.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/rapids/shims/GpuShuffleExchangeExec.scala index 39ac3fd30e5..178e8aa3c68 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/rapids/shims/GpuShuffleExchangeExec.scala +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/rapids/shims/GpuShuffleExchangeExec.scala @@ -35,6 +35,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.rapids.shims diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastExec.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastExec.scala similarity index 96% rename from sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastExec.scala rename to sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastExec.scala index 1861a9f2515..4708b3c19c0 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastExec.scala +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastExec.scala @@ -14,6 +14,32 @@ * limitations under the License. */ +/*** spark-rapids-shim-json-lines +{"spark": "311"} +{"spark": "312"} +{"spark": "313"} +{"spark": "320"} +{"spark": "321"} +{"spark": "321cdh"} +{"spark": "321db"} +{"spark": "322"} +{"spark": "323"} +{"spark": "324"} +{"spark": "330"} +{"spark": "330cdh"} +{"spark": "330db"} +{"spark": "331"} +{"spark": "332"} +{"spark": "332cdh"} +{"spark": "332db"} +{"spark": "333"} +{"spark": "340"} +{"spark": "341"} +{"spark": "341db"} +{"spark": "342"} +{"spark": "350"} +{"spark": "351"} +spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.execution import java.util.concurrent.{Future => JFuture} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala index 21463506f97..e0a8fa23ba4 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala @@ -38,6 +38,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.execution.python.shims diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala index be92a516bef..7ce270fcef2 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala @@ -38,6 +38,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.execution.python.shims diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupedPythonRunnerFactory.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupedPythonRunnerFactory.scala index 4186effcf84..49a475ea473 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupedPythonRunnerFactory.scala +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupedPythonRunnerFactory.scala @@ -35,6 +35,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.execution.python.shims diff --git a/sql-plugin/src/main/spark320/java/com/nvidia/spark/rapids/shims/ShimSupportsRuntimeFiltering.java b/sql-plugin/src/main/spark320/java/com/nvidia/spark/rapids/shims/ShimSupportsRuntimeFiltering.java index 14d200c7f48..09ceb189118 100644 --- a/sql-plugin/src/main/spark320/java/com/nvidia/spark/rapids/shims/ShimSupportsRuntimeFiltering.java +++ b/sql-plugin/src/main/spark320/java/com/nvidia/spark/rapids/shims/ShimSupportsRuntimeFiltering.java @@ -36,6 +36,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} + {"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims; diff --git a/sql-plugin/src/main/spark320/java/com/nvidia/spark/rapids/shims/XxHash64Shims.scala b/sql-plugin/src/main/spark320/java/com/nvidia/spark/rapids/shims/XxHash64Shims.scala index 9c38c67db93..3638675080a 100644 --- a/sql-plugin/src/main/spark320/java/com/nvidia/spark/rapids/shims/XxHash64Shims.scala +++ b/sql-plugin/src/main/spark320/java/com/nvidia/spark/rapids/shims/XxHash64Shims.scala @@ -36,6 +36,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/FileSourceScanExecMeta.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/FileSourceScanExecMeta.scala index ff6c408f7d8..db6c08f2486 100644 --- a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/FileSourceScanExecMeta.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/FileSourceScanExecMeta.scala @@ -32,6 +32,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuBatchScanExecBase.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuBatchScanExecBase.scala index 5002d4020e5..a3f71240b71 100644 --- a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuBatchScanExecBase.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuBatchScanExecBase.scala @@ -36,6 +36,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuOrcDataReader.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuOrcDataReader.scala index 9053d7e7cc2..b0261bbd145 100644 --- a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuOrcDataReader.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuOrcDataReader.scala @@ -32,6 +32,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuOrcDataReaderBase.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuOrcDataReaderBase.scala index 54f5e8d9ebf..21ec64a22e2 100644 --- a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuOrcDataReaderBase.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuOrcDataReaderBase.scala @@ -35,6 +35,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuParquetCrypto.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuParquetCrypto.scala index 7d2782886bc..16b860234c0 100644 --- a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuParquetCrypto.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuParquetCrypto.scala @@ -32,6 +32,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/HashUtils.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/HashUtils.scala index 72c2305d48b..ef81b9d52cd 100644 --- a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/HashUtils.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/HashUtils.scala @@ -35,6 +35,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/OffsetWindowFunctionMeta.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/OffsetWindowFunctionMeta.scala index 42354474c58..c38c7da06a1 100644 --- a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/OffsetWindowFunctionMeta.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/OffsetWindowFunctionMeta.scala @@ -36,6 +36,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/OrcCastingShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/OrcCastingShims.scala index 17a1970f3a2..07e060a1c66 100644 --- a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/OrcCastingShims.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/OrcCastingShims.scala @@ -33,6 +33,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/OrcShims320untilAllBase.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/OrcShims320untilAllBase.scala index 1e8ec738ae7..be812aeccef 100644 --- a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/OrcShims320untilAllBase.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/OrcShims320untilAllBase.scala @@ -33,6 +33,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/PlanShimsImpl.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/PlanShimsImpl.scala index dccfa4b23ff..f374f616d89 100644 --- a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/PlanShimsImpl.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/PlanShimsImpl.scala @@ -35,6 +35,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/RapidsCsvScanMeta.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/RapidsCsvScanMeta.scala index d1a2f81f57e..473a0a1ee53 100644 --- a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/RapidsCsvScanMeta.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/RapidsCsvScanMeta.scala @@ -36,6 +36,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/RebaseShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/RebaseShims.scala index 013051a809f..fe3d0fc109b 100644 --- a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/RebaseShims.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/RebaseShims.scala @@ -35,6 +35,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ShimAQEShuffleReadExec.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ShimAQEShuffleReadExec.scala index c2b4050aafb..fff6bd188ae 100644 --- a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ShimAQEShuffleReadExec.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ShimAQEShuffleReadExec.scala @@ -36,6 +36,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ShimBroadcastExchangeLike.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ShimBroadcastExchangeLike.scala index e65e2c8d907..03a020d772f 100644 --- a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ShimBroadcastExchangeLike.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ShimBroadcastExchangeLike.scala @@ -32,6 +32,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ShimPredicateHelper.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ShimPredicateHelper.scala index c54598719a0..6b1ef30e0de 100644 --- a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ShimPredicateHelper.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ShimPredicateHelper.scala @@ -36,6 +36,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/Spark320PlusNonDBShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/Spark320PlusNonDBShims.scala index d6691882663..b6b3f158c56 100644 --- a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/Spark320PlusNonDBShims.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/Spark320PlusNonDBShims.scala @@ -31,6 +31,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/Spark320PlusShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/Spark320PlusShims.scala index 072efe003d2..cad30c7a504 100644 --- a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/Spark320PlusShims.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/Spark320PlusShims.scala @@ -36,6 +36,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/TreeNode.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/TreeNode.scala index 07cf079491f..e3590443b76 100644 --- a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/TreeNode.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/TreeNode.scala @@ -36,6 +36,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/TypeSigUtil.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/TypeSigUtil.scala index 079be847a65..935b1eb6a94 100644 --- a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/TypeSigUtil.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/TypeSigUtil.scala @@ -36,6 +36,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/YearParseUtil.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/YearParseUtil.scala index 813987528b3..b4dc978b40e 100644 --- a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/YearParseUtil.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/YearParseUtil.scala @@ -35,6 +35,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/gpuWindows.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/gpuWindows.scala index 1fb2139411d..d4b9832842e 100644 --- a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/gpuWindows.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/gpuWindows.scala @@ -36,6 +36,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/v1FallbackWriters.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/v1FallbackWriters.scala index ea3883e0948..cc98b6ace50 100644 --- a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/v1FallbackWriters.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/v1FallbackWriters.scala @@ -36,6 +36,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids diff --git a/sql-plugin/src/main/spark320/scala/org/apache/spark/rapids/shims/GpuShuffleBlockResolver.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/rapids/shims/GpuShuffleBlockResolver.scala index 6a9cd609fd9..44a92ecfedd 100644 --- a/sql-plugin/src/main/spark320/scala/org/apache/spark/rapids/shims/GpuShuffleBlockResolver.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/rapids/shims/GpuShuffleBlockResolver.scala @@ -36,6 +36,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/org/apache/spark/rapids/shims/ShuffledBatchRDDUtil.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/rapids/shims/ShuffledBatchRDDUtil.scala index 38168912ffb..f828d7d0eac 100644 --- a/sql-plugin/src/main/spark320/scala/org/apache/spark/rapids/shims/ShuffledBatchRDDUtil.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/rapids/shims/ShuffledBatchRDDUtil.scala @@ -36,6 +36,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/org/apache/spark/rapids/shims/storage/ShimDiskBlockManager.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/rapids/shims/storage/ShimDiskBlockManager.scala index 5e50e667f15..222f06e0197 100644 --- a/sql-plugin/src/main/spark320/scala/org/apache/spark/rapids/shims/storage/ShimDiskBlockManager.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/rapids/shims/storage/ShimDiskBlockManager.scala @@ -36,6 +36,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.rapids.shims.storage diff --git a/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonOutput.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonOutput.scala index fde797c6965..ccd08de935e 100644 --- a/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonOutput.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonOutput.scala @@ -35,6 +35,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.execution.python.shims diff --git a/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/AvroUtils.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/AvroUtils.scala index 8633026c654..8a047cba60e 100644 --- a/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/AvroUtils.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/AvroUtils.scala @@ -36,6 +36,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/RapidsShuffleThreadedWriter.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/RapidsShuffleThreadedWriter.scala index 382523f0196..c616bda0477 100644 --- a/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/RapidsShuffleThreadedWriter.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/RapidsShuffleThreadedWriter.scala @@ -36,6 +36,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/Spark32XShimsUtils.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/Spark32XShimsUtils.scala index 0168fe766c7..276689b51fd 100644 --- a/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/Spark32XShimsUtils.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/Spark32XShimsUtils.scala @@ -36,6 +36,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/datetimeExpressions.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/datetimeExpressions.scala index fdb45beeb74..3f38d1f6a75 100644 --- a/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/datetimeExpressions.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/datetimeExpressions.scala @@ -36,6 +36,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.shims diff --git a/sql-plugin/src/main/spark320/scala/org/apache/spark/storage/RapidsPushBasedFetchHelper.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/storage/RapidsPushBasedFetchHelper.scala index 08435e3b02e..ca03bcfb7d1 100644 --- a/sql-plugin/src/main/spark320/scala/org/apache/spark/storage/RapidsPushBasedFetchHelper.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/storage/RapidsPushBasedFetchHelper.scala @@ -36,6 +36,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.storage diff --git a/sql-plugin/src/main/spark320/scala/org/apache/spark/storage/RapidsShuffleBlockFetcherIterator.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/storage/RapidsShuffleBlockFetcherIterator.scala index 572588b2707..c16202da2f7 100644 --- a/sql-plugin/src/main/spark320/scala/org/apache/spark/storage/RapidsShuffleBlockFetcherIterator.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/storage/RapidsShuffleBlockFetcherIterator.scala @@ -36,6 +36,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.storage diff --git a/sql-plugin/src/main/spark321/scala/com/nvidia/spark/rapids/shims/Spark321PlusShims.scala b/sql-plugin/src/main/spark321/scala/com/nvidia/spark/rapids/shims/Spark321PlusShims.scala index 452a3a1e832..6a1a4c25396 100644 --- a/sql-plugin/src/main/spark321/scala/com/nvidia/spark/rapids/shims/Spark321PlusShims.scala +++ b/sql-plugin/src/main/spark321/scala/com/nvidia/spark/rapids/shims/Spark321PlusShims.scala @@ -35,6 +35,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark321db/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala b/sql-plugin/src/main/spark321db/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala index 8f85ae8380e..280336012d0 100644 --- a/sql-plugin/src/main/spark321db/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala +++ b/sql-plugin/src/main/spark321db/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala @@ -30,6 +30,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/GpuBloomFilter.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/GpuBloomFilter.scala index 4274b24fc6b..f8a19b64cee 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/GpuBloomFilter.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/GpuBloomFilter.scala @@ -29,6 +29,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/GpuBloomFilterMightContain.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/GpuBloomFilterMightContain.scala index 15cb6a52676..e7ed75f24a2 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/GpuBloomFilterMightContain.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/GpuBloomFilterMightContain.scala @@ -29,6 +29,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/GpuInSubqueryExec.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/GpuInSubqueryExec.scala index 4d229d547a1..f0bbb11b410 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/GpuInSubqueryExec.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/GpuInSubqueryExec.scala @@ -26,6 +26,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/AnsiUtil.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/AnsiUtil.scala index 2d57fc2d950..71ce416b7fa 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/AnsiUtil.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/AnsiUtil.scala @@ -29,6 +29,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/BloomFilterShims.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/BloomFilterShims.scala index 251f3eeb175..38385ee5f08 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/BloomFilterShims.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/BloomFilterShims.scala @@ -29,6 +29,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/CharVarcharUtilsShims.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/CharVarcharUtilsShims.scala index 6e4944b969f..ac0ff9f7c50 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/CharVarcharUtilsShims.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/CharVarcharUtilsShims.scala @@ -29,6 +29,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/DayTimeIntervalShims.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/DayTimeIntervalShims.scala index 5d4bece1cba..34cce4db476 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/DayTimeIntervalShims.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/DayTimeIntervalShims.scala @@ -29,6 +29,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/DistributionUtil.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/DistributionUtil.scala index 7d26b91e4be..109dc58e73a 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/DistributionUtil.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/DistributionUtil.scala @@ -26,6 +26,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/FilteredPartitions.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/FilteredPartitions.scala index 4836b1e3b4d..127bf987b86 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/FilteredPartitions.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/FilteredPartitions.scala @@ -29,6 +29,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/GpuDataSourceRDD.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/GpuDataSourceRDD.scala index a84b8f0f3a3..dcda8ea9d4b 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/GpuDataSourceRDD.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/GpuDataSourceRDD.scala @@ -29,6 +29,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/GpuHashPartitioning.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/GpuHashPartitioning.scala index c6fc7d9a174..d52ac504487 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/GpuHashPartitioning.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/GpuHashPartitioning.scala @@ -29,6 +29,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/GpuIntervalUtilsBase.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/GpuIntervalUtilsBase.scala index 56bf9203c07..882856c4fdf 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/GpuIntervalUtilsBase.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/GpuIntervalUtilsBase.scala @@ -28,6 +28,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/GpuRangePartitioning.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/GpuRangePartitioning.scala index 889aa414f6b..e02f63b8009 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/GpuRangePartitioning.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/GpuRangePartitioning.scala @@ -29,6 +29,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/GpuTypeShims.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/GpuTypeShims.scala index 7b52ef9efec..2ea567afd12 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/GpuTypeShims.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/GpuTypeShims.scala @@ -28,6 +28,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/InSubqueryShims.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/InSubqueryShims.scala index 94ca8f0a913..c6b769d722c 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/InSubqueryShims.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/InSubqueryShims.scala @@ -26,6 +26,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/OrcReadingShims.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/OrcReadingShims.scala index cc8e69869c4..584dca41289 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/OrcReadingShims.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/OrcReadingShims.scala @@ -29,6 +29,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/OrcShims.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/OrcShims.scala index cc09214d8b1..4431dbf313d 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/OrcShims.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/OrcShims.scala @@ -26,6 +26,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/ParquetFieldIdShims.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/ParquetFieldIdShims.scala index 20e483baee3..99acc92c67a 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/ParquetFieldIdShims.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/ParquetFieldIdShims.scala @@ -28,6 +28,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/ParquetSchemaClipShims.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/ParquetSchemaClipShims.scala index 21f84bfe8b9..6e5e3e34237 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/ParquetSchemaClipShims.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/ParquetSchemaClipShims.scala @@ -29,6 +29,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/RapidsFileSourceMetaUtils.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/RapidsFileSourceMetaUtils.scala index 865ea22daa2..337aba375e3 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/RapidsFileSourceMetaUtils.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/RapidsFileSourceMetaUtils.scala @@ -29,6 +29,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/RapidsOrcScanMeta.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/RapidsOrcScanMeta.scala index 5a4eb2db9fb..a3208733c2d 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/RapidsOrcScanMeta.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/RapidsOrcScanMeta.scala @@ -29,6 +29,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/RapidsParquetScanMeta.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/RapidsParquetScanMeta.scala index 1bb4c4d4e6a..ce3ef417335 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/RapidsParquetScanMeta.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/RapidsParquetScanMeta.scala @@ -29,6 +29,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/RoundingShims.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/RoundingShims.scala index 5e943bcaa16..7f3556130ff 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/RoundingShims.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/RoundingShims.scala @@ -29,6 +29,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/ScanExecShims.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/ScanExecShims.scala index e0f22f16440..46f885dbed3 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/ScanExecShims.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/ScanExecShims.scala @@ -29,6 +29,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/Spark330PlusNonDBShims.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/Spark330PlusNonDBShims.scala index 1864b6dca38..fc75a537fdc 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/Spark330PlusNonDBShims.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/Spark330PlusNonDBShims.scala @@ -26,6 +26,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/Spark330PlusShims.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/Spark330PlusShims.scala index fe6d9fb4c05..2366dd7b409 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/Spark330PlusShims.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/Spark330PlusShims.scala @@ -26,6 +26,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/execution/datasources/parquet/ShimCurrentBatchIterator.scala b/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/execution/datasources/parquet/ShimCurrentBatchIterator.scala index 27cce9b3072..fa88b35d3e0 100644 --- a/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/execution/datasources/parquet/ShimCurrentBatchIterator.scala +++ b/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/execution/datasources/parquet/ShimCurrentBatchIterator.scala @@ -29,6 +29,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.execution.datasources.parquet diff --git a/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/aggregate/GpuBloomFilterAggregate.scala b/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/aggregate/GpuBloomFilterAggregate.scala index 512e159ac93..c4ee51b7b28 100644 --- a/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/aggregate/GpuBloomFilterAggregate.scala +++ b/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/aggregate/GpuBloomFilterAggregate.scala @@ -29,6 +29,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.aggregate diff --git a/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/GpuPythonMapInArrowExec.scala b/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/GpuPythonMapInArrowExec.scala index 6d06a97db06..c2d5083ed31 100644 --- a/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/GpuPythonMapInArrowExec.scala +++ b/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/GpuPythonMapInArrowExec.scala @@ -29,6 +29,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.execution.python diff --git a/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtilsFor330plus.scala b/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtilsFor330plus.scala index 81eb26e3078..2a486a14fd8 100644 --- a/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtilsFor330plus.scala +++ b/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtilsFor330plus.scala @@ -29,6 +29,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/RapidsShuffleThreadedReader.scala b/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/RapidsShuffleThreadedReader.scala index 6d25fb4da29..2fb5412ac45 100644 --- a/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/RapidsShuffleThreadedReader.scala +++ b/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/RapidsShuffleThreadedReader.scala @@ -29,6 +29,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/intervalExpressions.scala b/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/intervalExpressions.scala index 39262064ef2..17a0051b85f 100644 --- a/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/intervalExpressions.scala +++ b/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/intervalExpressions.scala @@ -29,6 +29,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.shims diff --git a/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/types/shims/PartitionValueCastShims.scala b/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/types/shims/PartitionValueCastShims.scala index 783417efb97..8fb94413435 100644 --- a/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/types/shims/PartitionValueCastShims.scala +++ b/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/types/shims/PartitionValueCastShims.scala @@ -29,6 +29,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.types.shims diff --git a/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/AnsiCastShim.scala b/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/AnsiCastShim.scala index 569ba8672f2..46872f5dc2c 100644 --- a/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/AnsiCastShim.scala +++ b/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/AnsiCastShim.scala @@ -21,6 +21,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/CastingConfigShim.scala b/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/CastingConfigShim.scala index 80d1209d575..044198f8458 100644 --- a/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/CastingConfigShim.scala +++ b/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/CastingConfigShim.scala @@ -22,6 +22,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/ColumnDefaultValuesShims.scala b/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/ColumnDefaultValuesShims.scala index ae118fd288d..71da82e5050 100644 --- a/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/ColumnDefaultValuesShims.scala +++ b/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/ColumnDefaultValuesShims.scala @@ -21,6 +21,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/DecimalArithmeticOverrides.scala b/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/DecimalArithmeticOverrides.scala index ba1df975366..17d2db8b90b 100644 --- a/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/DecimalArithmeticOverrides.scala +++ b/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/DecimalArithmeticOverrides.scala @@ -22,6 +22,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/GetMapValueMeta.scala b/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/GetMapValueMeta.scala index b6531828f70..dd9c447e046 100644 --- a/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/GetMapValueMeta.scala +++ b/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/GetMapValueMeta.scala @@ -22,6 +22,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/GpuCastShims.scala b/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/GpuCastShims.scala index 1d8aeda4b63..f87b8c2ada0 100644 --- a/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/GpuCastShims.scala +++ b/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/GpuCastShims.scala @@ -22,6 +22,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/ParquetStringPredShims.scala b/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/ParquetStringPredShims.scala index e1f3ddbd43f..4f8e41f6e52 100644 --- a/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/ParquetStringPredShims.scala +++ b/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/ParquetStringPredShims.scala @@ -22,6 +22,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/ShimFilePartitionReaderFactory.scala b/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/ShimFilePartitionReaderFactory.scala index c52f2b8498d..a0dc0afd3a0 100644 --- a/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/ShimFilePartitionReaderFactory.scala +++ b/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/ShimFilePartitionReaderFactory.scala @@ -22,6 +22,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/TypeUtilsShims.scala b/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/TypeUtilsShims.scala index 6ed589eaffa..8f7e872b83b 100644 --- a/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/TypeUtilsShims.scala +++ b/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/TypeUtilsShims.scala @@ -22,6 +22,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/execution/datasources/rapids/DataSourceStrategyUtils.scala b/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/execution/datasources/rapids/DataSourceStrategyUtils.scala index 04f0b220b5b..364406dca78 100644 --- a/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/execution/datasources/rapids/DataSourceStrategyUtils.scala +++ b/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/execution/datasources/rapids/DataSourceStrategyUtils.scala @@ -22,6 +22,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.execution.datasources.rapids diff --git a/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/GpuCheckOverflowInTableInsert.scala b/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/GpuCheckOverflowInTableInsert.scala index 07e713abcdf..00a17e1b3a4 100644 --- a/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/GpuCheckOverflowInTableInsert.scala +++ b/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/GpuCheckOverflowInTableInsert.scala @@ -27,6 +27,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids diff --git a/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/aggregate/aggregateFunctions.scala b/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/aggregate/aggregateFunctions.scala index 40cc9b418b8..d0eaa6b1be7 100644 --- a/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/aggregate/aggregateFunctions.scala +++ b/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/aggregate/aggregateFunctions.scala @@ -22,6 +22,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.aggregate diff --git a/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/arithmetic.scala b/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/arithmetic.scala index 7913602d2bf..15df31b9305 100644 --- a/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/arithmetic.scala +++ b/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/arithmetic.scala @@ -22,6 +22,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids diff --git a/sql-plugin/src/main/spark331/scala/com/nvidia/spark/rapids/shims/Spark331PlusNonDBShims.scala b/sql-plugin/src/main/spark331/scala/com/nvidia/spark/rapids/shims/Spark331PlusNonDBShims.scala index 046e71dbaa8..3755fcddca6 100644 --- a/sql-plugin/src/main/spark331/scala/com/nvidia/spark/rapids/shims/Spark331PlusNonDBShims.scala +++ b/sql-plugin/src/main/spark331/scala/com/nvidia/spark/rapids/shims/Spark331PlusNonDBShims.scala @@ -24,6 +24,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/CreateDataSourceTableAsSelectCommandMetaShims.scala b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/CreateDataSourceTableAsSelectCommandMetaShims.scala index 9240a226ecf..2a388a0d089 100644 --- a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/CreateDataSourceTableAsSelectCommandMetaShims.scala +++ b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/CreateDataSourceTableAsSelectCommandMetaShims.scala @@ -21,6 +21,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/FileIndexOptionsShims.scala b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/FileIndexOptionsShims.scala index 9802376697b..26ac4f3831e 100644 --- a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/FileIndexOptionsShims.scala +++ b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/FileIndexOptionsShims.scala @@ -21,6 +21,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala index 3da3abea1eb..e322efea6c1 100644 --- a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala +++ b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala @@ -21,6 +21,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.hive.rapids.shims diff --git a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuKnownNullable.scala b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuKnownNullable.scala index e0bd36696bd..84929c139e5 100644 --- a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuKnownNullable.scala +++ b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuKnownNullable.scala @@ -21,6 +21,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala index f1abe457800..48bd2a42012 100644 --- a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala +++ b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala @@ -21,6 +21,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/execution/datasources/GpuWriteFiles.scala b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/execution/datasources/GpuWriteFiles.scala index ec48578d56d..936b2472e8c 100644 --- a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/execution/datasources/GpuWriteFiles.scala +++ b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/execution/datasources/GpuWriteFiles.scala @@ -21,6 +21,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.execution.datasources diff --git a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/hive/rapids/shims/GpuRowBasedHiveGenericUDFShim.scala b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/hive/rapids/shims/GpuRowBasedHiveGenericUDFShim.scala index 78b0a216ae2..2aa1965c12c 100644 --- a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/hive/rapids/shims/GpuRowBasedHiveGenericUDFShim.scala +++ b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/hive/rapids/shims/GpuRowBasedHiveGenericUDFShim.scala @@ -18,6 +18,7 @@ {"spark": "332db"} {"spark": "341db"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.hive.rapids.shims diff --git a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/hive/rapids/shims/HiveFileUtil.scala b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/hive/rapids/shims/HiveFileUtil.scala index 681cf79e1e7..7d90e7a36f5 100644 --- a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/hive/rapids/shims/HiveFileUtil.scala +++ b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/hive/rapids/shims/HiveFileUtil.scala @@ -22,6 +22,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.hive.rapids.shims diff --git a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/hive/rapids/shims/HiveProviderCmdShims.scala b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/hive/rapids/shims/HiveProviderCmdShims.scala index 64b0eea741f..92e9282de4a 100644 --- a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/hive/rapids/shims/HiveProviderCmdShims.scala +++ b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/hive/rapids/shims/HiveProviderCmdShims.scala @@ -21,6 +21,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.hive.rapids.shims diff --git a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala index c683f71552a..3b8508dd6dc 100644 --- a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala +++ b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala @@ -21,6 +21,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids diff --git a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/GpuCreateDataSourceTableAsSelectCommandShims.scala b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/GpuCreateDataSourceTableAsSelectCommandShims.scala index b8080ffb810..a6bd907c8df 100644 --- a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/GpuCreateDataSourceTableAsSelectCommandShims.scala +++ b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/GpuCreateDataSourceTableAsSelectCommandShims.scala @@ -21,6 +21,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.shims diff --git a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/GpuDataSource.scala b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/GpuDataSource.scala index e8140b318b1..a4c81294a16 100644 --- a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/GpuDataSource.scala +++ b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/GpuDataSource.scala @@ -21,6 +21,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids diff --git a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/SchemaUtilsShims.scala b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/SchemaUtilsShims.scala index f99b4e41931..3f6d753e9e7 100644 --- a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/SchemaUtilsShims.scala +++ b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/SchemaUtilsShims.scala @@ -21,6 +21,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.shims diff --git a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/SparkDateTimeExceptionShims.scala b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/SparkDateTimeExceptionShims.scala index 58aba9822e3..fafaa14427a 100644 --- a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/SparkDateTimeExceptionShims.scala +++ b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/SparkDateTimeExceptionShims.scala @@ -21,6 +21,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.shims diff --git a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/SparkUpgradeExceptionShims.scala b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/SparkUpgradeExceptionShims.scala index 582a473ca34..d8670ad624d 100644 --- a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/SparkUpgradeExceptionShims.scala +++ b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/SparkUpgradeExceptionShims.scala @@ -21,6 +21,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.shims diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/CastCheckShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/CastCheckShims.scala index 74dab178deb..bdca4bac55f 100644 --- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/CastCheckShims.scala +++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/CastCheckShims.scala @@ -21,6 +21,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/GlobalLimitShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/GlobalLimitShims.scala index d8cb80d1d2f..b5fdf815a28 100644 --- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/GlobalLimitShims.scala +++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/GlobalLimitShims.scala @@ -20,6 +20,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/GpuBatchScanExec.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/GpuBatchScanExec.scala index 4d8b45e038b..25041dd85c7 100644 --- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/GpuBatchScanExec.scala +++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/GpuBatchScanExec.scala @@ -20,6 +20,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/GpuBroadcastJoinMeta.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/GpuBroadcastJoinMeta.scala index 7ed79194f8d..8f341f9cbea 100644 --- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/GpuBroadcastJoinMeta.scala +++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/GpuBroadcastJoinMeta.scala @@ -18,6 +18,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/GpuJsonToStructsShim.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/GpuJsonToStructsShim.scala index c05ebd2fa7c..d7016bae778 100644 --- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/GpuJsonToStructsShim.scala +++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/GpuJsonToStructsShim.scala @@ -19,6 +19,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/OrcProtoWriterShim.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/OrcProtoWriterShim.scala index b93f43417b6..8431d0eed4b 100644 --- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/OrcProtoWriterShim.scala +++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/OrcProtoWriterShim.scala @@ -21,6 +21,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ParquetLegacyNanoAsLongShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ParquetLegacyNanoAsLongShims.scala index ad6355d8281..3f48b9fbd2c 100644 --- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ParquetLegacyNanoAsLongShims.scala +++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ParquetLegacyNanoAsLongShims.scala @@ -20,6 +20,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ParquetTimestampAnnotationShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ParquetTimestampAnnotationShims.scala index 89ff642e6d3..7d8e7ed6ed1 100644 --- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ParquetTimestampAnnotationShims.scala +++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ParquetTimestampAnnotationShims.scala @@ -20,6 +20,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ParquetTimestampNTZShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ParquetTimestampNTZShims.scala index e0b73482e3f..ecc61d357fe 100644 --- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ParquetTimestampNTZShims.scala +++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ParquetTimestampNTZShims.scala @@ -20,6 +20,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala index 8aab7f55adf..9fb88817808 100644 --- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala +++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala @@ -19,6 +19,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ShuffleOriginUtil.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ShuffleOriginUtil.scala index 3369f813f72..2dbbf28832b 100644 --- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ShuffleOriginUtil.scala +++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ShuffleOriginUtil.scala @@ -19,6 +19,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/Spark340PlusNonDBShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/Spark340PlusNonDBShims.scala index a4af7318028..f60b3ba9e95 100644 --- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/Spark340PlusNonDBShims.scala +++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/Spark340PlusNonDBShims.scala @@ -19,6 +19,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/TagScanForRuntimeFiltering.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/TagScanForRuntimeFiltering.scala index 5c80cdf1407..d94b854f882 100644 --- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/TagScanForRuntimeFiltering.scala +++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/TagScanForRuntimeFiltering.scala @@ -20,6 +20,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleIterator.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleIterator.scala index 6e715e22d92..97ac1206f89 100644 --- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleIterator.scala +++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleIterator.scala @@ -20,6 +20,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shuffle diff --git a/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/catalyst/csv/GpuCsvUtils.scala b/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/catalyst/csv/GpuCsvUtils.scala index 393b7efb244..8b77e1a3cd8 100644 --- a/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/catalyst/csv/GpuCsvUtils.scala +++ b/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/catalyst/csv/GpuCsvUtils.scala @@ -20,6 +20,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.catalyst.csv diff --git a/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/catalyst/json/GpuJsonUtils.scala b/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/catalyst/json/GpuJsonUtils.scala index fa8649cf1d2..c37f0adc52d 100644 --- a/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/catalyst/json/GpuJsonUtils.scala +++ b/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/catalyst/json/GpuJsonUtils.scala @@ -20,6 +20,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.catalyst.json diff --git a/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/rapids/RapidsCachingReader.scala b/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/rapids/RapidsCachingReader.scala index 3694af18c18..80e530873f6 100644 --- a/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/rapids/RapidsCachingReader.scala +++ b/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/rapids/RapidsCachingReader.scala @@ -20,6 +20,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids diff --git a/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastHashJoinExec.scala b/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastHashJoinExec.scala index 22125e829a9..8a692ecc039 100644 --- a/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastHashJoinExec.scala +++ b/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastHashJoinExec.scala @@ -19,6 +19,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.execution diff --git a/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastNestedLoopJoinExec.scala b/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastNestedLoopJoinExec.scala index 9ff5acb5b16..ad2ce7fe443 100644 --- a/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastNestedLoopJoinExec.scala +++ b/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastNestedLoopJoinExec.scala @@ -19,6 +19,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.execution diff --git a/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/rapids/execution/ShimTrampolineUtil.scala b/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/rapids/execution/ShimTrampolineUtil.scala index 8eb5aa9453f..210a11984e7 100644 --- a/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/rapids/execution/ShimTrampolineUtil.scala +++ b/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/rapids/execution/ShimTrampolineUtil.scala @@ -19,6 +19,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.execution diff --git a/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala b/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala index 30d18974793..435a9a6f02c 100644 --- a/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala +++ b/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala @@ -19,6 +19,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.shims diff --git a/sql-plugin/src/main/spark341db/scala/com/nvidia/spark/rapids/shims/GpuAggregateInPandasExecMeta.scala b/sql-plugin/src/main/spark341db/scala/com/nvidia/spark/rapids/shims/GpuAggregateInPandasExecMeta.scala index 6873237d244..9332d897d09 100644 --- a/sql-plugin/src/main/spark341db/scala/com/nvidia/spark/rapids/shims/GpuAggregateInPandasExecMeta.scala +++ b/sql-plugin/src/main/spark341db/scala/com/nvidia/spark/rapids/shims/GpuAggregateInPandasExecMeta.scala @@ -17,6 +17,7 @@ /*** spark-rapids-shim-json-lines {"spark": "341db"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark341db/scala/com/nvidia/spark/rapids/shims/GpuToPrettyString.scala b/sql-plugin/src/main/spark341db/scala/com/nvidia/spark/rapids/shims/GpuToPrettyString.scala index 23ff476ebc6..d57c2b6a50d 100644 --- a/sql-plugin/src/main/spark341db/scala/com/nvidia/spark/rapids/shims/GpuToPrettyString.scala +++ b/sql-plugin/src/main/spark341db/scala/com/nvidia/spark/rapids/shims/GpuToPrettyString.scala @@ -17,6 +17,7 @@ /*** spark-rapids-shim-json-lines {"spark": "341db"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark341db/scala/com/nvidia/spark/rapids/shims/PythonUDFShim.scala b/sql-plugin/src/main/spark341db/scala/com/nvidia/spark/rapids/shims/PythonUDFShim.scala index e79730fc13f..667f556ef05 100644 --- a/sql-plugin/src/main/spark341db/scala/com/nvidia/spark/rapids/shims/PythonUDFShim.scala +++ b/sql-plugin/src/main/spark341db/scala/com/nvidia/spark/rapids/shims/PythonUDFShim.scala @@ -17,6 +17,7 @@ /*** spark-rapids-shim-json-lines {"spark": "341db"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/execution/rapids/shims/SplitFiles.scala b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/execution/rapids/shims/SplitFiles.scala index c76ea303ea4..f03e09fe4cd 100644 --- a/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/execution/rapids/shims/SplitFiles.scala +++ b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/execution/rapids/shims/SplitFiles.scala @@ -18,6 +18,7 @@ /*** spark-rapids-shim-json-lines {"spark": "341db"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.execution.rapids.shims diff --git a/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/hive/rapids/shims/CreateFunctions.scala b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/hive/rapids/shims/CreateFunctions.scala index c7c64830447..b82b5296da4 100644 --- a/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/hive/rapids/shims/CreateFunctions.scala +++ b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/hive/rapids/shims/CreateFunctions.scala @@ -17,6 +17,7 @@ /*** spark-rapids-shim-json-lines {"spark": "341db"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.hive.rapids.shims diff --git a/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/hive/rapids/shims/FileSinkDescShim.scala b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/hive/rapids/shims/FileSinkDescShim.scala index 1d43bf40206..886580332dc 100644 --- a/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/hive/rapids/shims/FileSinkDescShim.scala +++ b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/hive/rapids/shims/FileSinkDescShim.scala @@ -18,6 +18,7 @@ /*** spark-rapids-shim-json-lines {"spark": "341db"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.hive.rapids.shims diff --git a/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/hive/rapids/shims/HiveInspectorsShim.scala b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/hive/rapids/shims/HiveInspectorsShim.scala index d6a30ae671b..d0aae352ddb 100644 --- a/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/hive/rapids/shims/HiveInspectorsShim.scala +++ b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/hive/rapids/shims/HiveInspectorsShim.scala @@ -18,6 +18,7 @@ /*** spark-rapids-shim-json-lines {"spark": "341db"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.hive.rapids.shims diff --git a/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuBasePythonRunner.scala b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuBasePythonRunner.scala index 13df7923cd4..40b42e5b1a1 100644 --- a/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuBasePythonRunner.scala +++ b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuBasePythonRunner.scala @@ -17,6 +17,7 @@ /*** spark-rapids-shim-json-lines {"spark": "341db"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.execution.python.shims diff --git a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/BatchScanExecMeta.scala b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/BatchScanExecMeta.scala index e08765dda97..2ed7686f31d 100644 --- a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/BatchScanExecMeta.scala +++ b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/BatchScanExecMeta.scala @@ -16,6 +16,7 @@ /*** spark-rapids-shim-json-lines {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/GpuIntervalUtils.scala b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/GpuIntervalUtils.scala index 744bd93b029..a99ad3f51c5 100644 --- a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/GpuIntervalUtils.scala +++ b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/GpuIntervalUtils.scala @@ -15,6 +15,7 @@ */ /*** spark-rapids-shim-json-lines {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/KeyGroupedPartitioningShim.scala b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/KeyGroupedPartitioningShim.scala index 1d238be84b0..c2eeee8e07d 100644 --- a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/KeyGroupedPartitioningShim.scala +++ b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/KeyGroupedPartitioningShim.scala @@ -15,6 +15,7 @@ */ /*** spark-rapids-shim-json-lines {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/LegacyBehaviorPolicyShim.scala b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/LegacyBehaviorPolicyShim.scala index 99728ef9e74..fdd955184f9 100644 --- a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/LegacyBehaviorPolicyShim.scala +++ b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/LegacyBehaviorPolicyShim.scala @@ -16,6 +16,7 @@ /*** spark-rapids-shim-json-lines {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/NullOutputStreamShim.scala b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/NullOutputStreamShim.scala index f910f951d7a..5f4ddbf17fa 100644 --- a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/NullOutputStreamShim.scala +++ b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/NullOutputStreamShim.scala @@ -16,6 +16,7 @@ /*** spark-rapids-shim-json-lines {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/PythonMapInArrowExecShims.scala b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/PythonMapInArrowExecShims.scala index 98ccc540613..51711027328 100644 --- a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/PythonMapInArrowExecShims.scala +++ b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/PythonMapInArrowExecShims.scala @@ -16,6 +16,7 @@ /*** spark-rapids-shim-json-lines {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/SparkShims.scala b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/SparkShims.scala index e13aeb55195..dcc77bbc8de 100644 --- a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/SparkShims.scala +++ b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/SparkShims.scala @@ -16,6 +16,7 @@ /*** spark-rapids-shim-json-lines {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/shims/ParquetCVShims.scala b/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/shims/ParquetCVShims.scala index c15dc7907bb..a53506a8155 100644 --- a/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/shims/ParquetCVShims.scala +++ b/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/shims/ParquetCVShims.scala @@ -16,6 +16,7 @@ /*** spark-rapids-shim-json-lines {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.execution.datasources.parquet diff --git a/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/shims/ShimVectorizedColumnReader.scala b/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/shims/ShimVectorizedColumnReader.scala index e495dba634a..aa44f472d35 100644 --- a/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/shims/ShimVectorizedColumnReader.scala +++ b/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/shims/ShimVectorizedColumnReader.scala @@ -16,6 +16,7 @@ /*** spark-rapids-shim-json-lines {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.execution.datasources.parquet.rapids.shims diff --git a/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/execution/datasources/v2/rapids/GpuAtomicCreateTableAsSelectExec.scala b/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/execution/datasources/v2/rapids/GpuAtomicCreateTableAsSelectExec.scala index a5faad5122f..b52c699fd38 100644 --- a/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/execution/datasources/v2/rapids/GpuAtomicCreateTableAsSelectExec.scala +++ b/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/execution/datasources/v2/rapids/GpuAtomicCreateTableAsSelectExec.scala @@ -16,6 +16,7 @@ /*** spark-rapids-shim-json-lines {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.execution.datasources.v2.rapids diff --git a/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/execution/datasources/v2/rapids/GpuAtomicReplaceTableAsSelectExec.scala b/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/execution/datasources/v2/rapids/GpuAtomicReplaceTableAsSelectExec.scala index d32ab59db95..eddb8fa9e93 100644 --- a/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/execution/datasources/v2/rapids/GpuAtomicReplaceTableAsSelectExec.scala +++ b/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/execution/datasources/v2/rapids/GpuAtomicReplaceTableAsSelectExec.scala @@ -16,6 +16,7 @@ /*** spark-rapids-shim-json-lines {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.execution.datasources.v2.rapids diff --git a/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/execution/rapids/shims/FilePartitionShims.scala b/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/execution/rapids/shims/FilePartitionShims.scala index cfed5b884c6..6a36fbffb9a 100644 --- a/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/execution/rapids/shims/FilePartitionShims.scala +++ b/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/execution/rapids/shims/FilePartitionShims.scala @@ -17,6 +17,7 @@ /*** spark-rapids-shim-json-lines {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.execution.rapids.shims diff --git a/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/rapids/execution/GpuShuffleMeta.scala b/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/rapids/execution/GpuShuffleMeta.scala index 4876d31bb12..d61be4458e9 100644 --- a/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/rapids/execution/GpuShuffleMeta.scala +++ b/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/rapids/execution/GpuShuffleMeta.scala @@ -16,6 +16,7 @@ /*** spark-rapids-shim-json-lines {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.execution diff --git a/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/rapids/shims/ArrowUtilsShim.scala b/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/rapids/shims/ArrowUtilsShim.scala index edd4bfab7fb..fcbdea63b5f 100644 --- a/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/rapids/shims/ArrowUtilsShim.scala +++ b/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/rapids/shims/ArrowUtilsShim.scala @@ -16,6 +16,7 @@ /*** spark-rapids-shim-json-lines {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.shims diff --git a/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/rapids/shims/DataTypeUtilsShim.scala b/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/rapids/shims/DataTypeUtilsShim.scala index c3918c6f72f..80c536e88f9 100644 --- a/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/rapids/shims/DataTypeUtilsShim.scala +++ b/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/rapids/shims/DataTypeUtilsShim.scala @@ -16,6 +16,7 @@ /*** spark-rapids-shim-json-lines {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.shims diff --git a/sql-plugin/src/main/spark350emr/scala/com/nvidia/spark/rapids/GpuOutputAdapterExec.scala b/sql-plugin/src/main/spark350emr/scala/com/nvidia/spark/rapids/GpuOutputAdapterExec.scala new file mode 100644 index 00000000000..80c74279601 --- /dev/null +++ b/sql-plugin/src/main/spark350emr/scala/com/nvidia/spark/rapids/GpuOutputAdapterExec.scala @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "350emr"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids + +import com.nvidia.spark.rapids.shims.ShimUnaryExecNode + +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet} +import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.execution.exchange.BaseOutputAdapterExec +import org.apache.spark.sql.vectorized.ColumnarBatch + +/** + * Trivial GPU implementation of Spark's [[OutputAdapterExec]] to extend [[GpuExec]] + */ +case class GpuOutputAdapterExec( + override val output: Seq[Attribute], + override val child: SparkPlan) extends ShimUnaryExecNode + with BaseOutputAdapterExec with GpuExec { + + // The adapter effectively produces new attributes that are not found in input attributes + override def producedAttributes: AttributeSet = outputSet -- inputSet + + override def withNewChildInternal(newChild: SparkPlan): SparkPlan = + copy(child = newChild) + + override def doCanonicalize(): SparkPlan = child.canonicalized + + override def outputBatching: CoalesceGoal = GpuExec.outputBatching(child) + + override protected def internalDoExecuteColumnar(): RDD[ColumnarBatch] = { + throw new IllegalStateException(s"Internal Error ${this.getClass} has column support" + + s" mismatch:\n$this") + } +} \ No newline at end of file diff --git a/sql-plugin/src/main/spark350emr/scala/com/nvidia/spark/rapids/shims/spark350emr/SparkShimServiceProvider.scala b/sql-plugin/src/main/spark350emr/scala/com/nvidia/spark/rapids/shims/spark350emr/SparkShimServiceProvider.scala new file mode 100644 index 00000000000..eb538d94e37 --- /dev/null +++ b/sql-plugin/src/main/spark350emr/scala/com/nvidia/spark/rapids/shims/spark350emr/SparkShimServiceProvider.scala @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "350emr"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims.spark350emr + +import com.nvidia.spark.rapids.SparkShimVersion + +object SparkShimServiceProvider { + val VERSION = SparkShimVersion(3, 5, 0) + val VERSIONNAMES = Seq(s"$VERSION") +} + +class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { + + override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION + + override def matchesVersion(version: String): Boolean = { + val shortenedVersion = if (version != null) version.split("-")(0) else null + SparkShimServiceProvider.VERSIONNAMES.contains(shortenedVersion) + } +} diff --git a/sql-plugin/src/main/spark350emr/scala/com/nvidia/spark/rapids/spark350/RapidsShuffleManager.scala b/sql-plugin/src/main/spark350emr/scala/com/nvidia/spark/rapids/spark350/RapidsShuffleManager.scala new file mode 100644 index 00000000000..2b54278a8c7 --- /dev/null +++ b/sql-plugin/src/main/spark350emr/scala/com/nvidia/spark/rapids/spark350/RapidsShuffleManager.scala @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "350emr"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.spark350emr + +import org.apache.spark.SparkConf +import org.apache.spark.sql.rapids.ProxyRapidsShuffleInternalManagerBase + +/** A shuffle manager optimized for the RAPIDS Plugin for Apache Spark. */ +sealed class RapidsShuffleManager( + conf: SparkConf, + isDriver: Boolean +) extends ProxyRapidsShuffleInternalManagerBase(conf, isDriver) diff --git a/sql-plugin/src/main/spark350emr/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastExec.scala b/sql-plugin/src/main/spark350emr/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastExec.scala new file mode 100644 index 00000000000..5963318a2f3 --- /dev/null +++ b/sql-plugin/src/main/spark350emr/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastExec.scala @@ -0,0 +1,304 @@ +/* + * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "350emr"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.rapids.execution + +import scala.collection.JavaConverters.asScalaIteratorConverter + +import com.nvidia.spark.rapids.{BaseExprMeta, DataFromReplacementRule, GpuColumnarToRowExec, GpuExec, GpuMetric, GpuOutputAdapterExec, RapidsConf, RapidsMeta, SparkPlanMeta, TargetSize} +import com.nvidia.spark.rapids.Arm.withResource +import com.nvidia.spark.rapids.GpuMetric.{COLLECT_TIME, DESCRIPTION_COLLECT_TIME, ESSENTIAL_LEVEL} +import com.nvidia.spark.rapids.shims.{ShimUnaryExecNode, SparkShimImpl} + +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, BoundReference, Cast, Expression, NamedExpression, UnsafeProjection} +import org.apache.spark.sql.catalyst.plans.QueryPlan +import org.apache.spark.sql.catalyst.plans.physical.{BroadcastMode, IdentityBroadcastMode} +import org.apache.spark.sql.execution.{AsyncSubqueryExec, OutputAdapterExec, SparkPlan, SubqueryBroadcastExec} +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec +import org.apache.spark.sql.execution.exchange.BroadcastExchangeExec +import org.apache.spark.sql.execution.joins.{HashedRelationBroadcastMode, HashJoin} +import org.apache.spark.sql.vectorized.ColumnarBatch + + +class GpuSubqueryBroadcastMeta( + s: SubqueryBroadcastExec, + conf: RapidsConf, + p: Option[RapidsMeta[_, _, _]], + r: DataFromReplacementRule) extends + SparkPlanMeta[SubqueryBroadcastExec](s, conf, p, r) { + + private var broadcastBuilder: () => SparkPlan = _ + + override val childExprs: Seq[BaseExprMeta[_]] = Nil + + override val childPlans: Seq[SparkPlanMeta[SparkPlan]] = Nil + + override def tagPlanForGpu(): Unit = s.child match { + + // For AQE off: + // + // The rule PlanDynamicPruningFilters will insert SubqueryBroadcast if there exists + // available broadcast exchange for reuse. The plan stack of SubqueryBroadcast: + // + // +- SubqueryBroadcast + // +- BroadcastExchange (can be reused) + // +- [executed subquery...] + // + // Since the GPU overrides rule has been applied on executedSubQuery, if the + // executedSubQuery can be replaced by GPU overrides, the plan stack becomes: + // + // +- SubqueryBroadcast + // +- BroadcastExchange + // +- GpuColumnarToRow + // +- [GPU overrides of executed subquery...] + // + // To reuse BroadcastExchange on the GPU, we shall transform above pattern into: + // + // +- GpuSubqueryBroadcast + // +- GpuBroadcastExchange (can be reused) + // +- [GPU overrides of executed subquery...] + // + case ex @ BroadcastExchangeExec(_, c2r: GpuColumnarToRowExec) => + val exMeta = new GpuBroadcastMeta(ex.copy(child = c2r.child), conf, p, r) + exMeta.tagForGpu() + if (exMeta.canThisBeReplaced) { + broadcastBuilder = () => exMeta.convertToGpu() + } else { + willNotWorkOnGpu("underlying BroadcastExchange can not run in the GPU.") + } + + // For AQE on: + // + // In Spark 320+, DPP can cooperate with AQE. The insertion of SubqueryBroadcast is + // similar with non-AQE circumstance. During the creation of AdaptiveSparkPlan, the + // rule PlanAdaptiveSubqueries insert an intermediate plan SubqueryAdaptiveBroadcast to + // preserve the initial physical plan of DPP subquery filters. During the optimization, + // the rule PlanAdaptiveDynamicPruningFilters inserts the SubqueryBroadcast as the parent + // of adaptive subqueries: + // + // +- SubqueryBroadcast + // +- AdaptiveSparkPlan (supportColumnar=false) + // +- == Initial Plan == + // BroadcastExchange + // +- [executed subquery...] + // + // Since AdaptiveSparkPlan can be explicitly set as a columnar plan from Spark 320+, + // we can simply build GpuSubqueryBroadcast on the base of columnar adaptive plans + // whose root plan are GpuBroadcastExchange: + // + // +- GpuSubqueryBroadcast + // +- AdaptiveSparkPlan (supportColumnar=true) + // +- == Final Plan == + // BroadcastQueryStage + // +- GpuBroadcastExchange (can be reused) + // +- [GPU overrides of executed subquery...] + // + case a: AdaptiveSparkPlanExec => + tagAdaptivePlanForGpu(a, () => + SparkShimImpl.columnarAdaptivePlan( + a, TargetSize(conf.gpuTargetBatchSizeBytes))) + + case outputAdapter: OutputAdapterExec => + outputAdapter.child match { + case a: AdaptiveSparkPlanExec => + tagAdaptivePlanForGpu(a, () => + GpuOutputAdapterExec(outputAdapter.output, + SparkShimImpl.columnarAdaptivePlan(a, + TargetSize(conf.gpuTargetBatchSizeBytes)))) + case _ => + willNotWorkOnGpu("the subquery to broadcast can not entirely run in the GPU.") + } + + case _ => + willNotWorkOnGpu("the subquery to broadcast can not entirely run in the GPU.") + } + + def tagAdaptivePlanForGpu(adaptivePlan: AdaptiveSparkPlanExec, + buildBroadcast: () => SparkPlan): Unit = { + SparkShimImpl.getAdaptiveInputPlan(adaptivePlan) match { + case ex: BroadcastExchangeExec => + val exMeta = new GpuBroadcastMeta(ex, conf, p, r) + exMeta.tagForGpu() + if (exMeta.canThisBeReplaced) { + broadcastBuilder = buildBroadcast + } else { + willNotWorkOnGpu("underlying BroadcastExchange can not run in the GPU.") + } + case _ => + throw new AssertionError("should not reach here") + } + } + + /** + * Simply returns the original plan. Because its only child, BroadcastExchange, doesn't + * need to change if SubqueryBroadcastExec falls back to the CPU. + */ + override def convertToCpu(): SparkPlan = s + + override def convertToGpu(): GpuExec = { + GpuSubqueryBroadcastExec(s.name, s.index, s.buildKeys, broadcastBuilder())( + getBroadcastModeKeyExprs) + } + + /** Extract the broadcast mode key expressions if there are any. */ + private def getBroadcastModeKeyExprs: Option[Seq[Expression]] = { + val broadcastMode = s.child match { + case b: BroadcastExchangeExec => + b.mode + case a: AdaptiveSparkPlanExec => + getBroadcastModeFromAdaptivePlan(a) + case adapter: OutputAdapterExec => + getBroadcastModeFromOutputAdapter(adapter) + case _ => + throw new UnsupportedOperationException("Unknown SparkPlan") + } + + broadcastMode match { + case HashedRelationBroadcastMode(keys, _) => Some(keys) + case IdentityBroadcastMode => None + case m => throw new UnsupportedOperationException(s"Unknown broadcast mode $m") + } + } + + private def getBroadcastModeFromAdaptivePlan(plan: AdaptiveSparkPlanExec): BroadcastMode = { + SparkShimImpl.getAdaptiveInputPlan(plan) match { + case b: BroadcastExchangeExec => + b.mode + case _ => + throw new AssertionError("should not reach here") + } + } + + private def getBroadcastModeFromOutputAdapter(adapter: OutputAdapterExec): BroadcastMode = { + adapter.child match { + case a: AdaptiveSparkPlanExec => + getBroadcastModeFromAdaptivePlan(a) + case _ => + throw new AssertionError("should not reach here") + } + } +} + + +case class GpuSubqueryBroadcastExec( + name: String, + index: Int, + buildKeys: Seq[Expression], + child: SparkPlan)(modeKeys: Option[Seq[Expression]]) + extends AsyncSubqueryExec with GpuExec with ShimUnaryExecNode { + + override def otherCopyArgs: Seq[AnyRef] = modeKeys :: Nil + + // As `SubqueryBroadcastExec`, `GpuSubqueryBroadcastExec` is only used with `InSubqueryExec`. + // No one would reference this output, so the exprId doesn't matter here. But it's important to + // correctly report the output length, so that `InSubqueryExec` can know it's the single-column + // execution mode, not multi-column. + override def output: Seq[Attribute] = { + val key = buildKeys(index) + val name = key match { + case n: NamedExpression => + n.name + case cast: Cast if cast.child.isInstanceOf[NamedExpression] => + cast.child.asInstanceOf[NamedExpression].name + case _ => + "key" + } + Seq(AttributeReference(name, key.dataType, key.nullable)()) + } + + override lazy val additionalMetrics: Map[String, GpuMetric] = Map( + "dataSize" -> createSizeMetric(ESSENTIAL_LEVEL, "data size"), + COLLECT_TIME -> createNanoTimingMetric(ESSENTIAL_LEVEL, DESCRIPTION_COLLECT_TIME)) + + override def doCanonicalize(): SparkPlan = { + val keys = buildKeys.map(k => QueryPlan.normalizeExpressions(k, child.output)) + GpuSubqueryBroadcastExec("dpp", index, keys, child.canonicalized)(modeKeys) + } + + protected override def doExecute(): RDD[InternalRow] = { + throw new UnsupportedOperationException( + "GpuSubqueryBroadcastExec does not support the execute() code path.") + } + + override protected def doExecuteCollect(): Array[InternalRow] = { + val broadcastBatch = child.executeBroadcast[Any]() + val result: Array[InternalRow] = broadcastBatch.value match { + case b: SerializeConcatHostBuffersDeserializeBatch => projectSerializedBatchToRows(b) + case b if SparkShimImpl.isEmptyRelation(b) => Array.empty + case b => throw new IllegalStateException(s"Unexpected broadcast type: ${b.getClass}") + } + + result + } + + override protected def internalDoExecuteColumnar(): RDD[ColumnarBatch] = { + throw new IllegalStateException(s"Internal Error ${this.getClass} has column support" + + s" mismatch:\n$this") + } + + private def projectSerializedBatchToRows( + serBatch: SerializeConcatHostBuffersDeserializeBatch): Array[InternalRow] = { + val beforeCollect = System.nanoTime() + + // Creates projection to extract target field from Row, as what Spark does. + // Note that unlike Spark, the GPU broadcast data has not applied the key expressions from + // the HashedRelation, so that is applied here if necessary to ensure the proper values + // are being extracted. The CPU already has the key projections applied in the broadcast + // data and thus does not have similar logic here. + val broadcastModeProject = modeKeys.map { keyExprs => + val keyExpr = if (GpuHashJoin.canRewriteAsLongType(buildKeys)) { + // in this case, there is only 1 key expression since it's a packed version that encompasses + // multiple integral values into a single long using bit logic. In CPU Spark, the broadcast + // would create a LongHashedRelation instead of a standard HashedRelation. + keyExprs.head + } else { + keyExprs(index) + } + UnsafeProjection.create(keyExpr) + } + + // Use the single output of the broadcast mode projection if it exists + val rowProjectIndex = if (broadcastModeProject.isDefined) 0 else index + val rowExpr = if (GpuHashJoin.canRewriteAsLongType(buildKeys)) { + // Since this is the expected output for a LongHashedRelation, we can extract the key from the + // long packed key using bit logic, using this method available in HashJoin to give us the + // correct key expression. + HashJoin.extractKeyExprAt(buildKeys, index) + } else { + BoundReference(rowProjectIndex, buildKeys(index).dataType, buildKeys(index).nullable) + } + val rowProject = UnsafeProjection.create(rowExpr) + + // Deserializes the batch on the host. Then, transforms it to rows and performs row-wise + // projection. We should NOT run any device operation on the driver node. + val result = withResource(serBatch.hostBatch) { hostBatch => + hostBatch.rowIterator().asScala.map { row => + val broadcastRow = broadcastModeProject.map(_(row)).getOrElse(row) + rowProject(broadcastRow).copy().asInstanceOf[InternalRow] + }.toArray // force evaluation so we don't close hostBatch too soon + } + + gpuLongMetric("dataSize") += serBatch.dataSize + gpuLongMetric(COLLECT_TIME) += System.nanoTime() - beforeCollect + + result + } +} diff --git a/tests/src/test/scala/com/nvidia/spark/rapids/StringFunctionSuite.scala b/tests/src/test/scala/com/nvidia/spark/rapids/StringFunctionSuite.scala index 3c3933946c5..34cee026d02 100644 --- a/tests/src/test/scala/com/nvidia/spark/rapids/StringFunctionSuite.scala +++ b/tests/src/test/scala/com/nvidia/spark/rapids/StringFunctionSuite.scala @@ -272,13 +272,13 @@ class StringOperatorsDiagnostics extends SparkQueryCompareTestSuite { val codepoint = TestCodepoints.validCodepointIndices(i) print(f"(${codepoint.toChar.toString} $codepoint[$codepoint%04x]) ($cpu_str ") - print(f"${cpu_str.map(_.toInt.formatted("%d")).mkString(",")}") + print(f"${cpu_str.map(c => "%d".format(c.toInt)).mkString(",")}") print("[") - print(f"${cpu_str.map(_.toInt.formatted("%04x")).mkString(",")}") + print(f"${cpu_str.map(c => "%04x".format(c.toInt)).mkString(",")}") print(f"]) ($gpu_str ") - print(f"${gpu_str.map(_.toInt.formatted("%d")).mkString(",")}") + print(f"${gpu_str.map(c => "%d".format(c.toInt)).mkString(",")}") print("["); - print(f"${gpu_str.map(_.toInt.formatted("%04x")).mkString(",")}") + print(f"${gpu_str.map(c => "%04x".format(c.toInt)).mkString(",")}") println("])"); } } diff --git a/tests/src/test/spark320/scala/com/nvidia/spark/rapids/shims/OrcStatisticShim.scala b/tests/src/test/spark320/scala/com/nvidia/spark/rapids/shims/OrcStatisticShim.scala index e655abc24e5..d232be02a81 100644 --- a/tests/src/test/spark320/scala/com/nvidia/spark/rapids/shims/OrcStatisticShim.scala +++ b/tests/src/test/spark320/scala/com/nvidia/spark/rapids/shims/OrcStatisticShim.scala @@ -33,6 +33,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/GpuInSubqueryExecSuite.scala b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/GpuInSubqueryExecSuite.scala index 22d9ea63146..15355caecc6 100644 --- a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/GpuInSubqueryExecSuite.scala +++ b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/GpuInSubqueryExecSuite.scala @@ -26,6 +26,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids diff --git a/tests/src/test/spark340/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleTestHelper.scala b/tests/src/test/spark340/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleTestHelper.scala index f0f88508449..15638d6a07b 100644 --- a/tests/src/test/spark340/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleTestHelper.scala +++ b/tests/src/test/spark340/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleTestHelper.scala @@ -20,6 +20,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shuffle diff --git a/tests/src/test/spark341db/scala/com/nvidia/spark/rapids/ToPrettyStringSuite.scala b/tests/src/test/spark341db/scala/com/nvidia/spark/rapids/ToPrettyStringSuite.scala index c53ed57b8a5..6aab187fcdd 100644 --- a/tests/src/test/spark341db/scala/com/nvidia/spark/rapids/ToPrettyStringSuite.scala +++ b/tests/src/test/spark341db/scala/com/nvidia/spark/rapids/ToPrettyStringSuite.scala @@ -17,6 +17,7 @@ /*** spark-rapids-shim-json-lines {"spark": "341db"} {"spark": "350"} +{"spark": "350emr"} {"spark": "351"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids diff --git a/tests/src/test/spark350emr/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastExecSuite.scala b/tests/src/test/spark350emr/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastExecSuite.scala new file mode 100644 index 00000000000..c3232425ea2 --- /dev/null +++ b/tests/src/test/spark350emr/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastExecSuite.scala @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2019-2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "350emr"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.rapids.execution + +import scala.concurrent.Await +import scala.concurrent.duration.Duration +import scala.reflect.ClassTag + +import com.nvidia.spark.rapids.SparkSessionHolder +import org.mockito.Mockito +import org.scalatest.BeforeAndAfter +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.mockito.MockitoSugar + +import org.apache.spark.SparkException +import org.apache.spark.broadcast.Broadcast +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.dsl.expressions._ +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.execution.LeafExecNode +import org.apache.spark.sql.execution.vectorized.OnHeapColumnVector +import org.apache.spark.sql.types.LongType +import org.apache.spark.sql.vectorized.ColumnarBatch + +class GpuSubqueryBroadcastExecSuite extends AnyFunSuite with BeforeAndAfter with Matchers { + + before { + SparkSessionHolder // Ensure we have an active SparkSession + } + + test("executeCollect") { + val batch1 = createLongBatch(1, 2, 3) + val batch2 = createLongBatch(4, 5, 6) + val subqueryBroadcast = createIntGpuSubqueryBroadcast(batch1, batch2) + subqueryBroadcast.executeCollect().map(_.getInt(0)) shouldBe Seq(1, 2, 3, 4, 5, 6) + } + + test("materialize") { + val batch1 = createLongBatch(1, 2, 3) + val subqueryBroadcast = createIntGpuSubqueryBroadcast(batch1) + + val future = subqueryBroadcast.materialize() + val result = Await.result(future, Duration.Inf) + result shouldBe an [Array[InternalRow]] + result.asInstanceOf[Array[InternalRow]].map(_.getInt(0)) shouldBe Seq(1, 2, 3) + } + + test("cancel") { + val batch1 = createLongBatch(1, 2, 3) + val subqueryBroadcast = createIntGpuSubqueryBroadcast(batch1) + + subqueryBroadcast.cancel() + val future = subqueryBroadcast.materialize() + val thrown = the [SparkException] thrownBy Await.result(future, Duration.Inf) + thrown.getMessage should include ("was cancelled") + } + + private def createIntGpuSubqueryBroadcast(batches: ColumnarBatch*): GpuSubqueryBroadcastExec = { + val output = Seq('key.int) + val broadcast = MockGpuBroadcastExchangeExec(output, batches) + GpuSubqueryBroadcastExec("ddp", 0, output, broadcast)(None) + } + + private def createLongBatch(values: Long*): ColumnarBatch = { + val vector = new OnHeapColumnVector(values.length, LongType) + vector.putLongs(0, values.length, values.toArray, 0) + new ColumnarBatch(Array(vector), values.length) + } +} + +case class MockGpuBroadcastExchangeExec( + output: Seq[Attribute], data: Seq[ColumnarBatch]) extends LeafExecNode { + + override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException + + override protected[sql] def doExecuteBroadcast[T](): Broadcast[T] = { + val batch = MockitoSugar.mock[SerializeConcatHostBuffersDeserializeBatch] + Mockito.doReturn(data.toArray, Nil: _*).when(batch).hostBatch + new MockBroadcast[SerializeConcatHostBuffersDeserializeBatch](id = 0, batch) + .asInstanceOf[Broadcast[T]] + } +} + +class MockBroadcast[T: ClassTag](id: Long, value: T) extends Broadcast[T](id) { + + override protected def getValue(): T = value + + override protected def doUnpersist(blocking: Boolean): Unit = + throw new UnsupportedOperationException + + override protected def doDestroy(blocking: Boolean): Unit = + throw new UnsupportedOperationException +} \ No newline at end of file