diff --git a/.github/workflows/example.yml b/.github/workflows/example.yml index 3c0da74697..a0f11c119e 100644 --- a/.github/workflows/example.yml +++ b/.github/workflows/example.yml @@ -65,5 +65,5 @@ jobs: SPARK_VERSION: ${{ matrix.spark }} SPARK_COMPAT_VERSION: ${{ matrix.spark-compat }} SEDONA_VERSION: ${{ matrix.sedona }} - run: (cd examples/spark-sql;mvn clean install -Dspark.version=${SPARK_VERSION} -Dspark.compat.version=${SPARK_COMPAT_VERSION} -Dsedona.version=${SEDONA_VERSION};java -jar target/sedona-spark-example-1.0.0.jar) - - run: (cd examples/flink-sql;mvn clean install;java -jar target/sedona-flink-example-1.0.0.jar) + run: (cd examples/spark-sql;mvn clean install -Dspark.version=${SPARK_VERSION} -Dspark.compat.version=${SPARK_COMPAT_VERSION} -Dsedona.version=${SEDONA_VERSION};java -jar target/sedona-spark-example-1.6.0.jar) + - run: (cd examples/flink-sql;mvn clean install;java -jar target/sedona-flink-example-1.6.0.jar) diff --git a/.gitignore b/.gitignore index 955529bd74..991f92b826 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,12 @@ -/target/ +target /.idea -/*.iml +*.iml /*.ipr /*.iws /.settings/ /.classpath /.project -/dependency-reduced-pom.xml +dependency-reduced-pom.xml /bin/ /doc/ /conf/ diff --git a/examples/flink-sql/pom.xml b/examples/flink-sql/pom.xml index 5ac89ed8be..2ea51d41d5 100644 --- a/examples/flink-sql/pom.xml +++ b/examples/flink-sql/pom.xml @@ -20,32 +20,28 @@ 4.0.0 + org.apache.sedona sedona-flink-example - 1.0.0 - + 1.6.0 + Sedona : Examples : Flink jar UTF-8 - 1.5.1-28.2 compile - 2.12 - 1.5.1 - 1.14.3 + 1.19.0 compile + 2.12 + 28.2 + 2.17.2 org.apache.sedona sedona-flink_${scala.compat.version} - ${sedona.version} - - - org.datasyslab - geotools-wrapper - ${geotools.version} + ${project.version} org.apache.flink @@ -56,28 +52,21 @@ org.apache.flink - flink-streaming-java_${scala.compat.version} - ${flink.version} - ${flink.scope} - - - - org.apache.flink - flink-connector-kafka_${scala.compat.version} + flink-streaming-java ${flink.version} ${flink.scope} org.apache.flink - flink-clients_${scala.compat.version} + flink-clients ${flink.version} ${flink.scope} org.apache.flink - flink-table-api-java-bridge_${scala.compat.version} + flink-table-api-java-bridge ${flink.version} ${flink.scope} @@ -100,10 +89,50 @@ ${flink.version} ${flink.scope} + + + + org.geotools + gt-referencing + ${geotools.version} + compile + + + + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + compile + + + + org.apache.logging.log4j + log4j-api + ${log4j.version} + compile + + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + compile + + + + + org.apache.logging.log4j + log4j-1.2-api + ${log4j.version} + compile + + org.apache.flink - flink-runtime-web_${scala.compat.version} + flink-runtime-web ${flink.version} test diff --git a/examples/flink-sql/src/main/java/FlinkExample.java b/examples/flink-sql/src/main/java/FlinkExample.java index d39c34b782..c59eb8125e 100644 --- a/examples/flink-sql/src/main/java/FlinkExample.java +++ b/examples/flink-sql/src/main/java/FlinkExample.java @@ -21,6 +21,7 @@ import org.apache.flink.table.api.EnvironmentSettings; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; + import org.apache.sedona.flink.SedonaFlinkRegistrator; import org.apache.sedona.flink.expressions.Constructors; @@ -43,17 +44,18 @@ public static void main(String[] args) { // Create a fake WKT string table source Table pointWktTable = Utils.createTextTable(env, tableEnv, Utils.createPointWKT(testDataSize), pointColNames); + // Create a geometry column - Table pointTable = pointWktTable.select(call(Constructors.ST_GeomFromWKT.class.getSimpleName(), - $(pointColNames[0])).as(pointColNames[0]), - $(pointColNames[1])); + Table pointTable = pointWktTable.select( + call("ST_GeomFromWKT", $(pointColNames[0])).as(pointColNames[0]), + $(pointColNames[1])); + // Create S2CellID pointTable = pointTable.select($(pointColNames[0]), $(pointColNames[1]), call("ST_S2CellIDs", $(pointColNames[0]), 6).as("s2id_array")); // Explode s2id array tableEnv.createTemporaryView("pointTable", pointTable); pointTable = tableEnv.sqlQuery("SELECT geom_point, name_point, s2id_point FROM pointTable CROSS JOIN UNNEST(pointTable.s2id_array) AS tmpTbl1(s2id_point)"); - pointTable.execute().print(); // Create a fake WKT string table source @@ -68,13 +70,16 @@ public static void main(String[] args) { // Explode s2id array tableEnv.createTemporaryView("polygonTable", polygonTable); polygonTable = tableEnv.sqlQuery("SELECT geom_polygon, name_polygon, s2id_polygon FROM polygonTable CROSS JOIN UNNEST(polygonTable.s2id_array) AS tmpTbl2(s2id_polygon)"); - polygonTable.execute().print(); + + // TODO: TableImpl.print() occurs EOF Exception due to https://issues.apache.org/jira/browse/FLINK-35406 + // Use polygonTable.execute().print() when FLINK-35406 is fixed. + polygonTable.execute().collect().forEachRemaining(row -> System.out.println(row)); // Join two tables by their S2 ids Table joinResult = pointTable.join(polygonTable).where($("s2id_point").isEqual($("s2id_polygon"))); // Optional: remove false positives - joinResult = joinResult.where("ST_Contains(geom_polygon, geom_point)"); - joinResult.execute().print(); + joinResult = joinResult.where(call("ST_Contains", $("geom_polygon"), $("geom_point"))); + joinResult.execute().collect().forEachRemaining(row -> System.out.println(row)); } } diff --git a/examples/flink-sql/src/main/resources/log4j2.properties b/examples/flink-sql/src/main/resources/log4j2.properties new file mode 100644 index 0000000000..9206863eda --- /dev/null +++ b/examples/flink-sql/src/main/resources/log4j2.properties @@ -0,0 +1,25 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +rootLogger.level = INFO +rootLogger.appenderRef.console.ref = ConsoleAppender + +appender.console.name = ConsoleAppender +appender.console.type = CONSOLE +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n diff --git a/examples/spark-sql/colocationMap.png b/examples/spark-sql/colocationMap.png deleted file mode 100644 index 3e065f648a..0000000000 Binary files a/examples/spark-sql/colocationMap.png and /dev/null differ diff --git a/examples/spark-sql/pom.xml b/examples/spark-sql/pom.xml index 7826d0c83a..903a88bf15 100644 --- a/examples/spark-sql/pom.xml +++ b/examples/spark-sql/pom.xml @@ -20,11 +20,11 @@ 4.0.0 + org.apache.sedona sedona-spark-example - 1.0.0 - - ${project.groupId}:${project.artifactId} + 1.6.0 + Sedona : Examples : Spark Maven Example for SedonaDB jar diff --git a/flink/pom.xml b/flink/pom.xml index 55b778be20..b823d5783c 100644 --- a/flink/pom.xml +++ b/flink/pom.xml @@ -34,7 +34,6 @@ ${skip.deploy.common.modules} - 1.19.0 provided diff --git a/pom.xml b/pom.xml index bb2c321151..6fb2545688 100644 --- a/pom.xml +++ b/pom.xml @@ -86,6 +86,7 @@ 3.0 2.17.2 + 1.19.0 1.7.36 2.0.0 4.1.1 @@ -525,7 +526,7 @@ false ${project.basedir}/src/main/scala ${project.basedir}/src/test/scala - ${project.basedir}/../scalastyle_config.xml + tools/maven/scalastyle_config.xml ${project.basedir}/target/scalastyle-output.xml UTF-8 diff --git a/spark/pom.xml b/spark/pom.xml index 09ece37ace..ac7750a2c5 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -107,19 +107,4 @@ - - - - - - org.scalastyle - scalastyle-maven-plugin - - ${project.basedir}/../../scalastyle_config.xml - - - - - - diff --git a/scalastyle_config.xml b/tools/maven/scalastyle_config.xml similarity index 100% rename from scalastyle_config.xml rename to tools/maven/scalastyle_config.xml