Skip to content

Commit

Permalink
kamu: Upgrade to spark 3.5
Browse files Browse the repository at this point in the history
  • Loading branch information
@sergiimk
sergiimk committed Feb 11, 2024
1 parent 6de52a7 commit 0cfc08d
Show file tree
Hide file tree
Showing 4 changed files with 38 additions and 42 deletions.
9 changes: 7 additions & 2 deletions KAMU.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,19 @@ Kamu is building its own version of Livy for two reasons:

First clone the repo and make sure you're on `kamu` branch.

Clean command:
```sh
mvn clean -Pthriftserver -Pspark3 -Pscala-2.12 -DskipITs -DskipTests
```

Build command:
```sh
mvn package -Pthriftserver -Pspark-3.0 -Pspark.version=3.0.0 -Pscala.version=2.12.13 -Pscala.binary.version=2.12 -DskipITs -DskipTests
mvn package -Pthriftserver -Pspark3 -Pscala-2.12 -DskipITs -DskipTests
```
... this will take a while.

Package will be produced under:

```sh
./assembly/target/apache-livy-{version}-kamu-bin.zip
./assembly/target/apache-livy-{version}-bin.zip
```
67 changes: 29 additions & 38 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,17 +82,15 @@
<hadoop.scope>compile</hadoop.scope>
<slf4j.version>1.7.36</slf4j.version>
<reload4j.version>1.2.25</reload4j.version>
<spark.scala-2.11.version>2.4.5</spark.scala-2.11.version>
<spark.scala-2.12.version>2.4.5</spark.scala-2.12.version>
<spark.version>${spark.scala-2.11.version}</spark.version>
<spark.version>3.5.0</spark.version>
<hive.version>3.0.0</hive.version>
<commons-codec.version>1.9</commons-codec.version>
<httpclient.version>4.5.13</httpclient.version>
<httpcore.version>4.4.4</httpcore.version>
<jackson.version>2.12.7</jackson.version>
<jackson-databind.version>2.12.7.1</jackson-databind.version>
<jackson.version>2.15.2</jackson.version>
<jackson-databind.version>2.15.2</jackson-databind.version>
<javax.servlet-api.version>3.1.0</javax.servlet-api.version>
<jetty.version>9.4.50.v20221201</jetty.version>
<jetty.version>9.4.52.v20230823</jetty.version>
<junit.version>4.13.1</junit.version>
<libthrift.version>0.9.3</libthrift.version>
<kryo.version>4.0.2</kryo.version>
Expand All @@ -102,7 +100,7 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<scalatest.version>3.0.8</scalatest.version>
<scalatra.version>2.6.5</scalatra.version>
<java.version>1.8</java.version>
<java.version>17</java.version>
<test.redirectToFile>true</test.redirectToFile>
<execution.root>${user.dir}</execution.root>
<spark.home>${execution.root}/dev/spark</spark.home>
Expand All @@ -111,11 +109,11 @@
<hadoop.major-minor.version>2.7</hadoop.major-minor.version>
<hadoop.version>2.7.3</hadoop.version>
<!-- scala2.11 -->
<scala.binary.version>2.11</scala.binary.version>
<scala.version>2.11.12</scala.version>
<scala.binary.version>2.12</scala.binary.version>
<scala.version>2.12.18</scala.version>
<!-- spark2 -->
<spark.version>2.4.5</spark.version>
<java.version>1.8</java.version>
<spark.version>3.5.0</spark.version>
<java.version>17</java.version>
<py4j.version>0.10.9</py4j.version>
<json4s.version>3.5.3</json4s.version>
<spark.bin.name>spark-${spark.version}-bin-hadoop${hadoop.major-minor.version}</spark.bin.name>
Expand Down Expand Up @@ -176,7 +174,7 @@
<maven.compiler.target>${java.version}</maven.compiler.target>

<!-- kamu -->
<sedona.version>1.0.1-incubating</sedona.version>
<sedona.version>1.5.1</sedona.version>
</properties>

<repositories>
Expand Down Expand Up @@ -691,45 +689,45 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.3</version>
<version>3.11.0</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.6</version>
<version>3.6.0</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.8</version>
<version>3.1.0</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.3.2</version>
<version>3.3.0</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.2</version>
<version>3.1.1</version>
</plugin>

<plugin>
<groupId>org.apache.felix</groupId>
<artifactId>maven-bundle-plugin</artifactId>
<version>2.4.0</version>
<version>4.2.0</version>
</plugin>

<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>1.10</version>
<version>3.5.0</version>
</plugin>

<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>4.3.0</version>
<version>4.8.0</version>
<executions>
<execution>
<goals>
Expand Down Expand Up @@ -772,19 +770,19 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-install-plugin</artifactId>
<version>2.5.2</version>
<version>3.1.1</version>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.1</version>
<version>3.5.0</version>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.1</version>
<version>3.1.2</version>
<configuration>
<environmentVariables>
<LIVY_TEST>true</LIVY_TEST>
Expand All @@ -811,7 +809,7 @@
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<version>2.0.0</version>
<version>2.2.0</version>
<configuration>
<environmentVariables>
<LIVY_TEST>true</LIVY_TEST>
Expand Down Expand Up @@ -855,13 +853,13 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-remote-resources-plugin</artifactId>
<version>1.5</version>
<version>3.1.0</version>
</plugin>

<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>license-maven-plugin</artifactId>
<version>1.13</version>
<version>2.4</version>
</plugin>

<plugin>
Expand All @@ -877,7 +875,7 @@
<plugin>
<groupId>com.googlecode.maven-download-plugin</groupId>
<artifactId>download-maven-plugin</artifactId>
<version>1.3.0</version>
<version>1.8.0</version>
</plugin>

</plugins>
Expand Down Expand Up @@ -1051,7 +1049,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.10</version>
<version>3.6.0</version>
<configuration>
<outputDirectory>${project.build.directory}/jars</outputDirectory>
<useSubDirectoryPerType>false</useSubDirectoryPerType>
Expand Down Expand Up @@ -1141,18 +1139,11 @@
<hadoop.version>2.7.3</hadoop.version>
</properties>
</profile>
<profile>
<id>scala-2.11</id>
<properties>
<scala.binary.version>2.11</scala.binary.version>
<scala.version>2.11.12</scala.version>
</properties>
</profile>
<profile>
<id>scala-2.12</id>
<properties>
<scala.binary.version>2.12</scala.binary.version>
<scala.version>2.12.15</scala.version>
<scala.version>2.12.18</scala.version>
</properties>
</profile>

Expand Down Expand Up @@ -1181,12 +1172,12 @@
<profile>
<id>spark3</id>
<properties>
<spark.version>3.2.3</spark.version>
<spark.version>3.5.0</spark.version>
<java.version>1.8</java.version>
<py4j.version>0.10.9.7</py4j.version>
<json4s.version>3.7.0-M11</json4s.version>
<netty.version>4.1.92.Final</netty.version>
<spark.bin.name>spark-${spark.version}-bin-hadoop${hadoop.major-minor.version}</spark.bin.name>
<spark.bin.name>spark-${spark.version}-bin-hadoop3</spark.bin.name>
<spark.bin.download.url>
https://archive.apache.org/dist/spark/spark-${spark.version}/${spark.bin.name}.tgz
</spark.bin.download.url>
Expand Down
2 changes: 1 addition & 1 deletion rsc/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@
<!-- kamu -->
<dependency>
<groupId>org.apache.sedona</groupId>
<artifactId>sedona-python-adapter-3.0_2.12</artifactId>
<artifactId>sedona-spark-shaded-3.5_${scala.binary.version}</artifactId>
<version>${sedona.version}</version>
<scope>provided</scope>
</dependency>
Expand Down
2 changes: 1 addition & 1 deletion thriftserver/session/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@
<!-- kamu -->
<dependency>
<groupId>org.apache.sedona</groupId>
<artifactId>sedona-python-adapter-3.0_2.12</artifactId>
<artifactId>sedona-spark-shaded-3.5_${scala.binary.version}</artifactId>
<version>${sedona.version}</version>
<scope>provided</scope>
</dependency>
Expand Down

0 comments on commit 0cfc08d

Please sign in to comment.