Skip to content

Commit fc9d7ab

Browse files
ekrivokonmaprekrivokonmapr
committed
[EZSPA-1073] Add ssibility to work with spark.rapids.shims-provider-override option (apache#1114)
[EZSPA-1073] Add ssibility to work with spark.rapids.shims-provider-override option --------- Co-authored-by: Egor Krivokon <>
1 parent ee6c546 commit fc9d7ab

File tree

3 files changed

+130
-0
lines changed

3 files changed

+130
-0
lines changed

‎assembly/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,11 @@
7979
<artifactId>classpath-filter_${scala.binary.version}</artifactId>
8080
<version>${project.version}</version>
8181
</dependency>
82+
<dependency>
83+
<groupId>org.apache.spark</groupId>
84+
<artifactId>nvidia-plugin_${scala.binary.version}</artifactId>
85+
<version>${project.version}</version>
86+
</dependency>
8287
<dependency>
8388
<groupId>org.apache.spark</groupId>
8489
<artifactId>hive-site-editor_${scala.binary.version}</artifactId>

‎common/nvidia-plugin/pom.xml‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one or more
4+
~ contributor license agreements. See the NOTICE file distributed with
5+
~ this work for additional information regarding copyright ownership.
6+
~ The ASF licenses this file to You under the Apache License, Version 2.0
7+
~ (the "License"); you may not use this file except in compliance with
8+
~ the License. You may obtain a copy of the License at
9+
~
10+
~ http://www.apache.org/licenses/LICENSE-2.0
11+
~
12+
~ Unless required by applicable law or agreed to in writing, software
13+
~ distributed under the License is distributed on an "AS IS" BASIS,
14+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
~ See the License for the specific language governing permissions and
16+
~ limitations under the License.
17+
-->
18+
19+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
20+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
21+
<modelVersion>4.0.0</modelVersion>
22+
<parent>
23+
<groupId>org.apache.spark</groupId>
24+
<artifactId>spark-parent_2.12</artifactId>
25+
<version>3.3.2.0-eep-SNAPSHOT</version>
26+
<relativePath>../../pom.xml</relativePath>
27+
</parent>
28+
29+
<artifactId>nvidia-plugin_2.12</artifactId>
30+
<packaging>jar</packaging>
31+
<name>Nvdia Rapids plugin integration</name>
32+
<url>http://spark.apache.org/</url>
33+
<properties>
34+
<sbt.project.name>nvidia-plugin</sbt.project.name>
35+
</properties>
36+
37+
38+
<dependencies>
39+
<dependency>
40+
<groupId>org.scala-lang</groupId>
41+
<artifactId>scala-library</artifactId>
42+
<version>${scala.version}</version>
43+
<scope>provided</scope>
44+
</dependency>
45+
<dependency>
46+
<groupId>org.apache.spark</groupId>
47+
<artifactId>spark-core_${scala.binary.version}</artifactId>
48+
<version>${project.version}</version>
49+
</dependency>
50+
<dependency>
51+
<groupId>com.nvidia</groupId>
52+
<artifactId>rapids-4-spark_2.12</artifactId>
53+
<version>23.06.0</version>
54+
<scope>provided</scope>
55+
</dependency>
56+
</dependencies>
57+
58+
<build>
59+
<plugins>
60+
<plugin>
61+
<groupId>org.apache.maven.plugins</groupId>
62+
<artifactId>maven-dependency-plugin</artifactId>
63+
<executions>
64+
<!-- When using SPARK_PREPEND_CLASSES Spark classes compiled locally don't use
65+
shaded deps. So here we store jars in their original form which are added
66+
when the classpath is computed. -->
67+
<execution>
68+
<id>copy-dependencies</id>
69+
<phase>package</phase>
70+
<goals>
71+
<goal>copy-dependencies</goal>
72+
</goals>
73+
<configuration>
74+
<outputDirectory>${project.build.directory}</outputDirectory>
75+
<overWriteReleases>false</overWriteReleases>
76+
<overWriteSnapshots>false</overWriteSnapshots>
77+
<overWriteIfNewer>true</overWriteIfNewer>
78+
<useSubDirectoryPerType>true</useSubDirectoryPerType>
79+
<includeArtifactIds>
80+
ant
81+
</includeArtifactIds>
82+
<silent>true</silent>
83+
</configuration>
84+
</execution>
85+
</executions>
86+
</plugin>
87+
</plugins>
88+
</build>
89+
</project>

‎common/nvidia-plugin/src/main/scala/org/apache/spark/provider/EEPShimServiceProvider.scala‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.spark.provider
19+
20+
import com.nvidia.spark.rapids.SparkShimVersion
21+
import org.apache.spark.SPARK_VERSION_SHORT
22+
23+
object EEPShimServiceProvider {
24+
val Array(major, minor, patch) = SPARK_VERSION_SHORT.split("\\.").map(_.toInt)
25+
val VERSION = SparkShimVersion(major, minor, patch)
26+
val VERSIONNAMES = Seq(s"$VERSION")
27+
}
28+
29+
class EEPShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider {
30+
31+
override def getShimVersion: SparkShimVersion = EEPShimServiceProvider.VERSION
32+
33+
override def matchesVersion(version: String): Boolean = {
34+
EEPShimServiceProvider.VERSIONNAMES.contains(version)
35+
}
36+
}

0 commit comments

Comments
 (0)