[Feat][Spark] Spark 3.3.x support as a Maven Profile

.github/workflows/spark.yaml

@@ -37,6 +37,17 @@ concurrency:
 jobs:
   GraphAr-spark:
     runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - mvn-profile: "datasources-32"
+            spark: "spark-3.2.2"
+            spark-hadoop: "spark-3.2.2-bin-hadoop3.2"
+          - mvn-profile: "datasources-33"
+            spark: "spark-3.3.4"
+            spark-hadoop: "spark-3.3.4-bin-hadoop3"
+
     steps:
     - uses: actions/checkout@v3
       with:
@@ -53,22 +64,24 @@ jobs:
       run: |
         export JAVA_HOME=${JAVA_HOME_11_X64}
         pushd spark
-        mvn --no-transfer-progress clean package -DskipTests -Dspotless.check.skip=true
+        echo "Build ${{ matrix.mvn-profile }}"
+        mvn --no-transfer-progress clean package -DskipTests -Dspotless.check.skip=true -P ${{ matrix.mvn-profile }}
         popd
 
     - name: Run test
       run: |
         export JAVA_HOME=${JAVA_HOME_11_X64}
         pushd spark
-        mvn --no-transfer-progress test -Dspotless.check.skip=true
+        echo "Test ${{ matrix.mvn-profile }}"
+        mvn --no-transfer-progress test -Dspotless.check.skip=true -P ${{ matrix.mvn-profile }}
         popd
 
     - name: Run Neo4j2GraphAr example
       run: |
         export JAVA_HOME=${JAVA_HOME_11_X64}
         pushd spark
-        scripts/get-spark-to-home.sh
-        export SPARK_HOME="${HOME}/spark-3.2.2-bin-hadoop3.2"
+        scripts/get-spark-to-home.sh ${{ matrix.spark }} ${{ matrix.spark-hadoop }}
+        export SPARK_HOME="${HOME}/${{ matrix.spark-hadoop }}"
         export PATH="${SPARK_HOME}/bin":"${PATH}"
 
         scripts/get-neo4j-to-home.sh
@@ -78,7 +91,7 @@ jobs:
 
         scripts/deploy-neo4j-movie-data.sh
 
-        scripts/build.sh
+        scripts/build.sh ${{ matrix.mvn-profile }}
 
         export NEO4J_USR="neo4j"
         export NEO4J_PWD="neo4j"
@@ -90,20 +103,20 @@ jobs:
 
         # stop and clean
         popd
-    
+
     - name: Run Nebula2GraphAr example
       run: |
         export JAVA_HOME=${JAVA_HOME_11_X64}
         pushd spark
         scripts/get-nebula-to-home.sh
-        export SPARK_HOME="${HOME}/spark-3.2.2-bin-hadoop3.2"
+        export SPARK_HOME="${HOME}/${{ matrix.spark-hadoop }}"
         export PATH="${SPARK_HOME}/bin":"${PATH}"
 
         scripts/get-nebula-to-home.sh
 
         scripts/deploy-nebula-default-data.sh
 
-        scripts/build.sh
+        scripts/build.sh ${{ matrix.mvn-profile }}
 
         scripts/run-nebula2graphar.sh
 
@@ -113,7 +126,7 @@ jobs:
             --name nebula-console-loader \
             --network nebula-docker-env_nebula-net \
             vesoft/nebula-console:nightly -addr 172.28.3.1 -port 9669 -u root -p nebula -e "use basketballplayer; clear space basketballplayer;"
-        
+
         # import from GraphAr
         scripts/run-graphar2nebula.sh
 
@@ -124,11 +137,10 @@ jobs:
       run: |
         export JAVA_HOME=${JAVA_HOME_11_X64}
         pushd spark
-        scripts/get-spark-to-home.sh
-        export SPARK_HOME="${HOME}/spark-3.2.2-bin-hadoop3.2"
+        export SPARK_HOME="${HOME}/${{ matrix.spark-hadoop }}"
         export PATH="${SPARK_HOME}/bin":"${PATH}"
 
-        scripts/build.sh
+        scripts/build.sh ${{ matrix.mvn-profile }}
 
         # run the importer
         cd import

.licenserc.yaml

@@ -37,6 +37,7 @@ header:
     - '**/.scalafmt.conf'
     - 'cpp/apidoc'
     - 'spark/datasources-32/src/main/scala/com/alibaba/graphar/datasources'
+    - 'spark/datasources-33/src/main/scala/com/alibaba/graphar/datasources'
     - '*.md'
     - '*.rst'
     - '**/*.json'

spark/datasources-33/.scalafmt.conf

@@ -0,0 +1 @@
+../.scalafmt.conf

spark/datasources-33/pom.xml

@@ -0,0 +1,188 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright 2022-2023 Alibaba Group Holding Limited.
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>com.alibaba</groupId>
+        <artifactId>graphar</artifactId>
+        <version>${graphar.version}</version>
+    </parent>
+
+    <groupId>com.alibaba</groupId>
+    <artifactId>graphar-datasources</artifactId>
+    <version>${graphar.version}</version>
+    <packaging>jar</packaging>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-core_${scala.binary.version}</artifactId>
+            <version>${spark.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-streaming_${scala.binary.version}</artifactId>
+            <version>${spark.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-mllib_${scala.binary.version}</artifactId>
+            <version>${spark.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-sql_${scala.binary.version}</artifactId>
+            <version>${spark.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-hive_${scala.binary.version}</artifactId>
+            <version>${spark.version}</version>
+            <scope>provided</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.scala-tools</groupId>
+                <artifactId>maven-scala-plugin</artifactId>
+                <version>2.15.2</version>
+                <configuration>
+                    <scalaVersion>${scala.version}</scalaVersion>
+                    <args>
+                        <arg>-target:jvm-1.8</arg>
+                    </args>
+                    <jvmArgs>
+                        <jvmArg>-Xss4096K</jvmArg>
+                    </jvmArgs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>scala-compile</id>
+                        <goals>
+                            <goal>compile</goal>
+                        </goals>
+                        <configuration>
+                            <excludes>
+                                <exclude>META-INF/*.SF</exclude>
+                                <exclude>META-INF/*.DSA</exclude>
+                                <exclude>META-INF/*.RSA</exclude>
+                            </excludes>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>scala-test-compile</id>
+                        <goals>
+                            <goal>testCompile</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>net.alchim31.maven</groupId>
+                <artifactId>scala-maven-plugin</artifactId>
+                <version>4.8.0</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>compile</goal>
+                            <goal>testCompile</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <jvmArgs>
+                        <jvmArg>-Xms64m</jvmArg>
+                        <jvmArg>-Xmx1024m</jvmArg>
+                    </jvmArgs>
+                    <args>
+                        <arg>-Ywarn-unused</arg>
+                    </args>
+                    <compilerPlugins>
+                        <compilerPlugin>
+                            <groupId>org.scalameta</groupId>
+                            <artifactId>semanticdb-scalac_2.12.10</artifactId>
+                            <version>4.3.24</version>
+                        </compilerPlugin>
+                    </compilerPlugins>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>com.diffplug.spotless</groupId>
+                <artifactId>spotless-maven-plugin</artifactId>
+                <version>2.20.0</version>
+                <configuration>
+                    <!-- define a language-specific format -->
+                    <java>
+                        <!-- no need to specify files, inferred automatically, but you can if you want -->
+                        <!-- apply a specific flavor of google-java-format and reflow long strings -->
+                        <googleJavaFormat>
+                            <version>1.13.0</version>
+                            <style>AOSP</style>
+                        </googleJavaFormat>
+                    </java>
+                    <scala>
+                        <scalafmt>
+                            <file>${project.basedir}/.scalafmt.conf</file> <!-- optional -->
+                        </scalafmt>
+                    </scala>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>io.github.evis</groupId>
+                <artifactId>scalafix-maven-plugin_2.13</artifactId>
+                <version>0.1.8_0.11.0</version>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-source-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>attach-sources</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-javadoc-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>attach-javadocs</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <artifactId>maven-site-plugin</artifactId>
+                <version>3.7.1</version>
+            </plugin>
+        </plugins>
+    </build>
+</project>

spark/datasources-33/src/main/java/com/alibaba/graphar/GeneralParams.java

@@ -0,0 +1 @@
+../../../../../../../graphar/src/main/java/com/alibaba/graphar/GeneralParams.java