diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index d32e88dc368b..880be2b755da 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -52,7 +52,7 @@ jobs: with: submodules: true - name: Check code style - run: ./mvnw --batch-mode --quiet --no-snapshot-updates clean checkstyle:check + run: ./mvnw --batch-mode --quiet --no-snapshot-updates clean spotless:check dead-link: if: github.repository == 'apache/incubator-seatunnel' @@ -101,9 +101,8 @@ jobs: cache: 'maven' - name: Build distribution tar run: >- - ./mvnw -B install scalastyle:check + ./mvnw -B install -D"maven.test.skip"=true - -D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true -D"http.keepAlive"=false -D"maven.wagon.http.pool"=false @@ -130,8 +129,6 @@ jobs: ./mvnw -B -q install -DskipTests -D"maven.test.skip"=true -D"maven.javadoc.skip"=true - -D"scalastyle.skip"=true - -D"checkstyle.skip"=true -D"license.skipAddThirdParty" - name: Check Dependencies Licenses run: tools/dependencies/checkLicense.sh @@ -155,7 +152,7 @@ jobs: cache: 'maven' - name: Run Unit tests run: | - ./mvnw -B -T 1C clean verify -D"maven.test.skip"=false -D"checkstyle.skip"=true -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates + ./mvnw -B -T 1C clean verify -D"maven.test.skip"=false -D"license.skipAddThirdParty"=true --no-snapshot-updates env: MAVEN_OPTS: -Xmx2048m @@ -178,6 +175,6 @@ jobs: cache: 'maven' - name: Run Integration tests run: | - ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"checkstyle.skip"=true -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates + ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates env: MAVEN_OPTS: -Xmx2048m diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index 237bebd08e15..4037603157c5 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -30,7 +30,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 env: - JAVA_TOOL_OPTIONS: -Xmx2G -Xms2G -Dhttp.keepAlive=false -Dmaven.test.skip=true -Dcheckstyle.skip=true -Dlicense.skipAddThirdParty=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + JAVA_TOOL_OPTIONS: -Xmx2G -Xms2G -Dhttp.keepAlive=false -Dmaven.test.skip=true -Dlicense.skipAddThirdParty=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 strategy: fail-fast: false diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1cf4140aa4f1..e994fd0f2e8c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -54,7 +54,6 @@ jobs: run: | ./mvnw -B package \ -Dmaven.test.skip=true \ - -Dcheckstyle.skip=true \ -Dlicense.skipAddThirdParty=true \ -Dhttp.keepAlive=false \ -Dmaven.wagon.http.pool=false \ diff --git a/.github/workflows/license.yml b/.github/workflows/license.yml index d9cd901cab89..6ef7f113703b 100644 --- a/.github/workflows/license.yml +++ b/.github/workflows/license.yml @@ -67,7 +67,7 @@ jobs: ${{ runner.os }}-maven- - name: Generate THIRD-PARTY run: | - ./mvnw clean license:aggregate-add-third-party -DskipTests -Dcheckstyle.skip -U + ./mvnw clean license:aggregate-add-third-party -DskipTests -U - name: Check LICENSE file run: | python3 tools/dependencies/license.py seatunnel-dist/target/THIRD-PARTY.txt seatunnel-dist/release-docs/LICENSE true diff --git a/.github/workflows/publish-docker.yaml b/.github/workflows/publish-docker.yaml index c2ffb5b433a2..e3b53331a5a4 100644 --- a/.github/workflows/publish-docker.yaml +++ b/.github/workflows/publish-docker.yaml @@ -68,7 +68,6 @@ jobs: ./mvnw -B clean deploy \ -Dmaven.test.skip \ -Dmaven.javadoc.skip \ - -Dmaven.checkstyle.skip \ -Dlicense.skipAddThirdParty=true \ -Dmaven.deploy.skip \ -Ddocker.tag=${{ github.sha }} \ diff --git a/docs/en/contribution/setup.md b/docs/en/contribution/setup.md index 5191d39ea81d..8b0feb868dfc 100644 --- a/docs/en/contribution/setup.md +++ b/docs/en/contribution/setup.md @@ -36,7 +36,9 @@ Otherwise, your code could not start in JetBrains IntelliJ IDEA correctly. ``` ### Building SeaTunnel from source + After you install the maven, you can use the follow command to compile and package. + ``` mvn clean package -pl seatunnel-dist -am -Dmaven.test.skip=true ``` @@ -52,13 +54,19 @@ See [install plugins for IDEA](https://www.jetbrains.com/help/idea/managing-plug Before running the following example, you should also install JetBrains IntelliJ IDEA's [Lombok plugin](https://plugins.jetbrains.com/plugin/6317-lombok). See [install plugins for IDEA](https://www.jetbrains.com/help/idea/managing-plugins.html#install-plugins) if you want to. -### Install JetBrains IDEA CheckStyle-IDEA Plugin +### Code Style -Before coding, you should also install JetBrains IntelliJ IDEA's [CheckStyle-IDEA plugin](https://plugins.jetbrains.com/plugin/1065-checkstyle-idea). -See [install plugins for IDEA](https://www.jetbrains.com/help/idea/managing-plugins.html#install-plugins) if you want to. -Next, you should go to `Preferences -> Editor -> Code style -> Scheme -> Import Scheme -> CheckStyle Configration` and import `tools/checkstyle/checkStyle.xml` -![checkstyle.png](../images/checkstyle.png) -If you want to change to automatically formatting, these configurations are also required. +SeaTunnel uses `Spotless` for code style and formatting checks. +You could run the following command and `Spotless` will automatically fix +the code style and formatting errors for you: + +```shell +./mvnw spotless:apply +``` + +You could copy the `pre-commit hook` file `/tools/style/pre-commit` to your `.git/hooks/` +directory so that every time you commit your code with `git commit`, `Spotless` will automatically +fix things for you. ## Run Simple Example @@ -69,7 +77,6 @@ it in IDEA](https://www.jetbrains.com/help/idea/run-debug-configuration.html) as Here we use `seatunnel-examples/seatunnel-flink-examples/src/main/java/org/apache/seatunnel/example/flink/LocalFlinkExample.java` as an example, when you run it successfully you could see the output as below: - ```log +I[Ricky Huo, 71] +I[Gary, 12] @@ -83,7 +90,7 @@ as an example, when you run it successfully you could see the output as below: All our examples use simple source and sink to make it less dependent and easy to run. You can change the example configuration in `resources/examples`. You could change your configuration as below, if you want to use PostgreSQL as the source and -sink to console. +sink to console. ```conf env { @@ -103,3 +110,4 @@ sink { ConsoleSink {} } ``` + diff --git a/pom.xml b/pom.xml index 6f17bdf72b68..c742c1f01ea6 100644 --- a/pom.xml +++ b/pom.xml @@ -13,8 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. --> - 4.0.0 @@ -26,50 +25,12 @@ org.apache.seatunnel seatunnel - pom ${revision} + pom SeaTunnel - - Production ready big data processing product based on Apache Spark and Apache Flink. - - - https://github.com/apache/incubator-seatunnel - - - - The Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt - - - - - scm:git:https://github.com/apache/incubator-seatunnel.git - scm:git:https://github.com/apache/incubator-seatunnel.git - https://github.com/apache/incubator-seatunnel - HEAD - - - - GitHub - https://github.com/apache/incubator-seatunnel/issues - - - - - SeaTunnel Developer List - dev@seatunnel.apache.org - dev-subscribe@seatunnel.apache.org - dev-unsubscribe@seatunnel.apache.org - - - SeaTunnel Commits List - commits@seatunnel.apache.org - commits-subscribe@seatunnel.apache.org - commits-unsubscribe@seatunnel.apache.org - - + Production ready big data processing product based on Apache Spark and Apache Flink. 2.1.3-SNAPSHOT @@ -151,8 +86,6 @@ false 2.22.2 2.22.2 - 3.1.2 - true 1.6.8 3.0.1 2.9.1 @@ -180,7 +113,6 @@ 1.3.0 1.20 2.17.1 - 1.0.0 0.38.0 1.3.0 1.9.5 @@ -287,8 +219,8 @@ slf4j-api - junit junit + junit @@ -421,17 +353,17 @@ scala-compile-first - process-resources compile + process-resources scala-test-compile - process-test-resources testCompile + process-test-resources @@ -460,8 +392,7 @@ ${skipUT} - ${project.build.directory}/jacoco.exec - + ${project.build.directory}/jacoco.exec **/*IT.java @@ -522,14 +453,14 @@ - package shade + package - + @@ -543,79 +474,6 @@ ${maven-assembly-plugin.version} - - - org.apache.maven.plugins - maven-checkstyle-plugin - ${maven-checkstyle-plugin.version} - - - ${maven.multiModuleProjectDirectory}/tools/checkstyle/checkStyle.xml - - UTF-8 - true - true - ${checkstyle.fails.on.error} - - ${project.build.sourceDirectory} - ${project.build.testSourceDirectory} - - - **/*.properties, - **/*.sh, - **/*.bat, - **/*.yml, - **/*.yaml, - **/*.xml - - - **/.asf.yaml, - **/.github/** - - - - - - - validate - process-sources - - check - - - - - - - org.scalastyle - scalastyle-maven-plugin - ${scalastyle-maven-plugin.version} - - false - true - false - false - ${basedir}/src/main/scala - ${basedir}/src/test/scala - - - ${maven.multiModuleProjectDirectory}/tools/checkstyle/scalastyle-config.xml - - ${project.build.directory}/target/scalastyle-output.xml - UTF-8 - UTF-8 - - - - - check - - - - - - - org.apache.maven.plugins maven-source-plugin @@ -651,7 +509,6 @@ - org.codehaus.mojo build-helper-maven-plugin @@ -687,17 +544,17 @@ flatten - process-resources flatten + process-resources flatten.clean - clean clean + clean @@ -743,16 +600,6 @@ maven-failsafe-plugin - - org.apache.maven.plugins - maven-checkstyle-plugin - - - - org.scalastyle - scalastyle-maven-plugin - - org.codehaus.mojo license-maven-plugin @@ -785,7 +632,8 @@ **/*.sc - + + @@ -827,4 +675,66 @@ + https://github.com/apache/incubator-seatunnel + + + + The Apache License, Version 2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + + + + + + SeaTunnel Developer List + dev-subscribe@seatunnel.apache.org + dev-unsubscribe@seatunnel.apache.org + dev@seatunnel.apache.org + + + SeaTunnel Commits List + commits-subscribe@seatunnel.apache.org + commits-unsubscribe@seatunnel.apache.org + commits@seatunnel.apache.org + + + + + scm:git:https://github.com/apache/incubator-seatunnel.git + scm:git:https://github.com/apache/incubator-seatunnel.git + https://github.com/apache/incubator-seatunnel + HEAD + + + + GitHub + https://github.com/apache/incubator-seatunnel/issues + + + + + all + + true + + release + false + + + + seatunnel-examples + seatunnel-e2e + + + + release + + + release + true + + + + + diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-console/src/main/scala/org/apache/seatunnel/spark/console/sink/Console.scala b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-console/src/main/scala/org/apache/seatunnel/spark/console/sink/Console.scala index 50ae94d438b5..5a1e874b0e04 100644 --- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-console/src/main/scala/org/apache/seatunnel/spark/console/sink/Console.scala +++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-console/src/main/scala/org/apache/seatunnel/spark/console/sink/Console.scala @@ -18,7 +18,15 @@ package org.apache.seatunnel.spark.console.sink import org.apache.seatunnel.common.config.CheckResult import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory -import org.apache.seatunnel.spark.console.Config.{LIMIT, SERIALIZER, PLAIN, JSON, SCHEMA, DEFAULT_SERIALIZER, DEFAULT_LIMIT} +import org.apache.seatunnel.spark.console.Config.{ + LIMIT, + SERIALIZER, + PLAIN, + JSON, + SCHEMA, + DEFAULT_SERIALIZER, + DEFAULT_LIMIT +} import org.apache.seatunnel.spark.SparkEnvironment import org.apache.seatunnel.spark.batch.SparkBatchSink import org.apache.spark.sql.{Dataset, Row} @@ -53,10 +61,15 @@ class Console extends SparkBatchSink { } override def checkConfig(): CheckResult = { - if (!config.hasPath(LIMIT) || (config.hasPath(LIMIT) && config.getInt(LIMIT) >= -1)) { + if ( + !config + .hasPath(LIMIT) || (config.hasPath(LIMIT) && config.getInt(LIMIT) >= -1) + ) { CheckResult.success() } else { - CheckResult.error("Please specify [" + LIMIT + "] as Number[-1, " + Int.MaxValue + "]") + CheckResult.error( + "Please specify [" + LIMIT + "] as Number[-1, " + Int.MaxValue + "]" + ) } } @@ -65,7 +78,8 @@ class Console extends SparkBatchSink { Map( LIMIT -> DEFAULT_LIMIT, SERIALIZER -> DEFAULT_SERIALIZER // plain | json - )) + ) + ) config = config.withFallback(defaultConfig) } diff --git a/tools/checkstyle/checkStyle.xml b/tools/checkstyle/checkStyle.xml deleted file mode 100755 index b6395cb6fd72..000000000000 --- a/tools/checkstyle/checkStyle.xml +++ /dev/null @@ -1,543 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/tools/checkstyle/scalastyle-config.xml b/tools/checkstyle/scalastyle-config.xml deleted file mode 100644 index b81d4421dc48..000000000000 --- a/tools/checkstyle/scalastyle-config.xml +++ /dev/null @@ -1,292 +0,0 @@ - - - - - Scalastyle standard configuration - - - - - - - - - - - - - - - - - - - - - - - - - - ^[a-z][A-Za-z0-9]*$ - - - - - - - - - - - - ^[a-z][A-Za-z0-9]*$ - ^$ - - - - - - 15 - - - - - - - - - - - - Illegal import of sun._ or java.awt._ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - true - true - - - - - - - - - - - - - \)\{ - - - - - - - - mutable\.SynchronizedBuffer - - - - - - - - Await\.result - - - - - - - - Await\.ready - - - - - - - - FileSystem.get\([a-zA-Z_$][a-zA-Z_$0-9]*\) - - - - - - - - org\.apache\.commons\.lang\. - - Use Commons Lang 3 classes (package org.apache.commons.lang3.*) instead - of Commons Lang 2 (package org.apache.commons.lang.*) - - - - - - ^println$ - - - - - - - - - - - ^.*$ - false - - - - - - - 800 - - - - - - - 50 - - - - - - - 80 - - - - - - - - - - - - - - - - - - - LPAREN - - - - - - ARROW, EQUALS, ELSE, TRY, CATCH, FINALLY, LARROW, RARROW - - - - - - ARROW, EQUALS, COMMA, COLON, IF, ELSE, DO, WHILE, FOR, MATCH, TRY, CATCH, FINALLY, - LARROW, RARROW - - - - - - - COLON, COMMA, RPAREN - - - - - - \ No newline at end of file diff --git a/tools/style/spotless_seatunnel_formatter.xml b/tools/style/spotless_seatunnel_formatter.xml index 4327ddc1fae4..aa30e4434e32 100644 --- a/tools/style/spotless_seatunnel_formatter.xml +++ b/tools/style/spotless_seatunnel_formatter.xml @@ -19,7 +19,7 @@ */ --> - +