Merge branch 'master' into SEDONA-668

apache · Oct 27, 2024 · 3edcaa2 · 3edcaa2
2 parents 12443f0 + 4841279
commit 3edcaa2
Show file tree

Hide file tree

Showing 39 changed files with 963 additions and 229 deletions.
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
@@ -41,45 +41,37 @@ jobs:
           - spark: '3.5.0'
             scala: '2.12.8'
             python: '3.10'
-            hadoop: '3'
             shapely: '1'
           - spark: '3.5.0'
             scala: '2.12.8'
             python: '3.10'
-            hadoop: '3'
           - spark: '3.5.0'
             scala: '2.12.8'
             python: '3.9'
-            hadoop: '3'
           - spark: '3.5.0'
             scala: '2.12.8'
             python: '3.8'
-            hadoop: '3'
           - spark: '3.4.0'
             scala: '2.12.8'
             python: '3.10'
-            hadoop: '3'
           - spark: '3.4.0'
             scala: '2.12.8'
             python: '3.9'
-            hadoop: '3'
           - spark: '3.4.0'
             scala: '2.12.8'
             python: '3.8'
-            hadoop: '3'
           - spark: '3.4.0'
             scala: '2.12.8'
             python: '3.7'
-            hadoop: '3'
           - spark: '3.4.0'
             scala: '2.12.8'
             python: '3.7'
-            hadoop: '3'
             shapely: '1'
           - spark: '3.3.0'
             scala: '2.12.8'
             python: '3.8'
-            hadoop: '3'
+    env:
+      VENV_PATH: /home/runner/.local/share/virtualenvs/python-${{ matrix.python }}
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-java@v4
@@ -101,18 +93,6 @@ jobs:
         run: |
             SPARK_COMPAT_VERSION=${SPARK_VERSION:0:3}
             mvn -q clean install -DskipTests -Dspark=${SPARK_COMPAT_VERSION} -Dscala=${SCALA_VERSION:0:4} -Dgeotools
-      - env:
-          SPARK_VERSION: ${{ matrix.spark }}
-          HADOOP_VERSION: ${{ matrix.hadoop }}
-        run: |
-          wget https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
-          wget https://repo.osgeo.org/repository/release/javax/media/jai_core/${JAI_CORE_VERSION}/jai_core-${JAI_CORE_VERSION}.jar
-          wget https://repo.osgeo.org/repository/release/javax/media/jai_codec/${JAI_CODEC_VERSION}/jai_codec-${JAI_CODEC_VERSION}.jar
-          wget https://repo.osgeo.org/repository/release/javax/media/jai_imageio/${JAI_IMAGEIO_VERSION}/jai_imageio-${JAI_IMAGEIO_VERSION}.jar
-          tar -xzf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
-          mv -v jai_core-${JAI_CORE_VERSION}.jar spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}/jars/
-          mv -v jai_codec-${JAI_CODEC_VERSION}.jar spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}/jars/
-          mv -v jai_imageio-${JAI_IMAGEIO_VERSION}.jar spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}/jars/
       - run: sudo apt-get -y install python3-pip python-dev-is-python3
       - run: sudo pip3 install -U setuptools
       - run: sudo pip3 install -U wheel
@@ -129,32 +109,43 @@ jobs:
             echo "Patching Pipfile to use Shapely 1.x"
             sed -i 's/^shapely.*$/shapely="<2.0.0"/g' Pipfile
           fi
+          export PIPENV_CUSTOM_VENV_NAME=python-${PYTHON_VERSION}
           pipenv --python ${PYTHON_VERSION}
           pipenv install pyspark==${SPARK_VERSION}
           pipenv install --dev
           pipenv graph
       - env:
-          SPARK_VERSION: ${{ matrix.spark }}
-          HADOOP_VERSION: ${{ matrix.hadoop }}
-        run: find spark-shaded/target -name sedona-*.jar -exec cp {} spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}/jars/ \;
+          PYTHON_VERSION: ${{ matrix.python }}
+        run: |
+          wget --retry-connrefused --waitretry=10 --read-timeout=20 --timeout=15 --tries=5 https://repo.osgeo.org/repository/release/javax/media/jai_core/${JAI_CORE_VERSION}/jai_core-${JAI_CORE_VERSION}.jar
+          wget --retry-connrefused --waitretry=10 --read-timeout=20 --timeout=15 --tries=5 https://repo.osgeo.org/repository/release/javax/media/jai_codec/${JAI_CODEC_VERSION}/jai_codec-${JAI_CODEC_VERSION}.jar
+          wget --retry-connrefused --waitretry=10 --read-timeout=20 --timeout=15 --tries=5 https://repo.osgeo.org/repository/release/javax/media/jai_imageio/${JAI_IMAGEIO_VERSION}/jai_imageio-${JAI_IMAGEIO_VERSION}.jar
+          mv -v jai_core-${JAI_CORE_VERSION}.jar ${VENV_PATH}/lib/python${PYTHON_VERSION}/site-packages/pyspark/jars
+          mv -v jai_codec-${JAI_CODEC_VERSION}.jar ${VENV_PATH}/lib/python${PYTHON_VERSION}/site-packages/pyspark/jars
+          mv -v jai_imageio-${JAI_IMAGEIO_VERSION}.jar ${VENV_PATH}/lib/python${PYTHON_VERSION}/site-packages/pyspark/jars
       - env:
-          SPARK_VERSION: ${{ matrix.spark }}
-          HADOOP_VERSION: ${{ matrix.hadoop }}
-        run: (export SPARK_HOME=$PWD/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION};export PYTHONPATH=$SPARK_HOME/python;cd python;pipenv run pytest tests)
+          PYTHON_VERSION: ${{ matrix.python }}
+        run: find spark-shaded/target -name sedona-*.jar -exec cp {} ${VENV_PATH}/lib/python${PYTHON_VERSION}/site-packages/pyspark/jars/ \;
       - env:
-          SPARK_VERSION: ${{ matrix.spark }}
-          HADOOP_VERSION: ${{ matrix.hadoop }}
+          PYTHON_VERSION: ${{ matrix.python }}
+        run: |
+          export SPARK_HOME=${VENV_PATH}/lib/python${PYTHON_VERSION}/site-packages/pyspark
+          cd python
+          source ${VENV_PATH}/bin/activate
+          pytest tests
+      - env:
+          PYTHON_VERSION: ${{ matrix.python }}
         run: |
-          if [ ! -f "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}/sbin/start-connect-server.sh" ]
+          if [ ! -f "${VENV_PATH}/lib/python${PYTHON_VERSION}/site-packages/pyspark/sbin/start-connect-server.sh" ]
           then
             echo "Skipping connect tests for Spark $SPARK_VERSION"
             exit
           fi
 
-          export SPARK_HOME=$PWD/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}
-          export PYTHONPATH=$SPARK_HOME/python
+          export SPARK_HOME=${VENV_PATH}/lib/python${PYTHON_VERSION}/site-packages/pyspark
           export SPARK_REMOTE=local
 
           cd python
-          pipenv install "pyspark[connect]==${SPARK_VERSION}"
-          pipenv run pytest tests/sql/test_dataframe_api.py
+          source ${VENV_PATH}/bin/activate
+          pip install "pyspark[connect]==${SPARK_VERSION}"
+          pytest tests/sql/test_dataframe_api.py
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -14,11 +14,11 @@ repos:
     rev: 24.10.0
     hooks:
       - id: black-jupyter
-  - repo: https://github.com/pycqa/isort
-    rev: 5.13.2
-    hooks:
-      - id: isort
-        name: isort (python)
+  # - repo: https://github.com/pycqa/isort
+  #   rev: 5.13.2
+  #   hooks:
+  #     - id: isort
+  #       name: isort (python)
   - repo: https://github.com/pre-commit/mirrors-clang-format
     rev: v19.1.1
     hooks:
@@ -78,6 +78,18 @@ repos:
       - id: forbid-submodules
       - id: mixed-line-ending
         exclude: \.csv$
+      - id: name-tests-test
+        args: [--pytest-test-first]
+        exclude: |
+          (?x)^(
+              python/tests/properties/crs_transform\.py|
+              python/tests/properties/linestring_properties\.py|
+              python/tests/properties/point_properties\.py|
+              python/tests/properties/polygon_properties\.py|
+              python/tests/sql/resource/sample_data\.py|
+              python/tests/streaming/spark/cases_builder\.py|
+              python/tests/tools\.py
+          )$
       - id: requirements-txt-fixer
         files: ^docker/sedona-spark-jupyterlab/requirements\.txt$
       - id: trailing-whitespace

diff --git a/common/src/main/java/org/apache/sedona/common/Functions.java b/common/src/main/java/org/apache/sedona/common/Functions.java
@@ -1050,7 +1050,7 @@ public static InscribedCircle maximumInscribedCircle(Geometry geometry) {
     // All non-polygonal geometries use LargestEmptyCircle
     if (!geometry.getClass().getSimpleName().equals("Polygon")
         && !geometry.getClass().getSimpleName().equals("MultiPolygon")) {
-      LargestEmptyCircle largestEmptyCircle = new LargestEmptyCircle(geometry, tolerance);
+      LargestEmptyCircle largestEmptyCircle = new LargestEmptyCircle(geometry, null, tolerance);
       center = largestEmptyCircle.getCenter();
       nearest = largestEmptyCircle.getRadiusPoint();
       radius = largestEmptyCircle.getRadiusLine().getLength();
@@ -2266,6 +2266,45 @@ public static Geometry points(Geometry geometry) {
     return geometry.getFactory().createMultiPointFromCoords(coordinates);
   }
 
+  public static Geometry scale(Geometry geometry, double scaleX, double scaleY) {
+    return scaleGeom(geometry, Constructors.point(scaleX, scaleY));
+  }
+
+  public static Geometry scaleGeom(Geometry geometry, Geometry factor) {
+    return scaleGeom(geometry, factor, null);
+  }
+
+  public static Geometry scaleGeom(Geometry geometry, Geometry factor, Geometry origin) {
+    if (geometry == null || factor == null || geometry.isEmpty() || factor.isEmpty()) {
+      return geometry;
+    }
+
+    if (!factor.getGeometryType().equalsIgnoreCase(Geometry.TYPENAME_POINT)) {
+      throw new IllegalArgumentException("Scale factor geometry should be a Point type.");
+    }
+
+    Geometry resultGeom = null;
+    AffineTransformation scaleInstance = null;
+    Coordinate factorCoordinate = factor.getCoordinate();
+
+    if (origin == null || origin.isEmpty()) {
+      scaleInstance =
+          AffineTransformation.scaleInstance(factorCoordinate.getX(), factorCoordinate.getY());
+      resultGeom = scaleInstance.transform(geometry);
+    } else {
+      Coordinate falseOrigin = origin.getCoordinate();
+      scaleInstance =
+          AffineTransformation.scaleInstance(
+              factorCoordinate.getX(),
+              factorCoordinate.getY(),
+              falseOrigin.getX(),
+              falseOrigin.getY());
+      resultGeom = scaleInstance.transform(geometry);
+    }
+
+    return resultGeom;
+  }
+
   public static Geometry rotateX(Geometry geometry, double angle) {
     if (GeomUtils.isAnyGeomEmpty(geometry)) {
       return geometry;

diff --git a/common/src/main/java/org/apache/sedona/common/utils/InscribedCircle.java b/common/src/main/java/org/apache/sedona/common/utils/InscribedCircle.java
@@ -45,4 +45,15 @@ public boolean equals(InscribedCircle other) {
         && this.nearest.equals(other.nearest)
         && Math.abs(this.radius - other.radius) < epsilon;
   }
+
+  public boolean equals(Object obj) {
+    if (obj == this) {
+      return true;
+    }
+    if (obj == null || obj.getClass() != this.getClass()) {
+      return false;
+    }
+    InscribedCircle other = (InscribedCircle) obj;
+    return this.equals(other);
+  }
 }
diff --git a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java
@@ -1598,7 +1598,7 @@ public void delaunayTriangles() throws ParseException {
     String actual = Functions.delaunayTriangle(combined).toText();
     String expected =
         "GEOMETRYCOLLECTION (POLYGON ((20 40, 125 100, 50 60, 20 40)), POLYGON ((20 40, 50 60, 110 170, 20 40)), POLYGON ((110 170, 50 60, 125 100, 110 170)), POLYGON ((110 170, 125 100, 175 150, 110 170)))";
-    assertEquals(expected, actual);
+    assertGeometryEquals(expected, actual);
 
     poly =
         Constructors.geomFromEWKT(
@@ -1610,7 +1610,7 @@ public void delaunayTriangles() throws ParseException {
     actual = Functions.delaunayTriangle(poly, 0, 1).toText();
     expected =
         "MULTILINESTRING ((25 20, 35 20), (20 20, 25 20), (10 20, 20 20), (10 10, 10 20), (10 10, 20 10), (20 10, 25 10), (25 10, 35 10), (35 10, 35 20), (25 20, 35 10), (25 10, 25 20), (20 20, 25 10), (20 10, 20 20), (10 20, 20 10))";
-    assertEquals(expected, actual);
+    assertGeometryEquals(expected, actual);
   }
 
   @Test
@@ -1802,14 +1802,11 @@ public void simplify() throws ParseException {
     expectedPoints = 17;
     assertEquals(expectedPoints, actualPoints);
 
-    actualPoints = Functions.nPoints(Functions.simplify(geom, 1));
+    Geometry actual = Functions.simplify(geom, 1);
+    actualPoints = Functions.nPoints(actual);
     expectedPoints = 9;
     assertEquals(expectedPoints, actualPoints);
 
-    Geometry actual = Functions.simplify(geom, 10);
-    actualPoints = Functions.nPoints(actual);
-    expectedPoints = 4;
-    assertEquals(expectedPoints, actualPoints);
     assertEquals(1111, actual.getSRID());
   }
 
@@ -3733,22 +3730,22 @@ public void testAddMeasure() throws ParseException {
   public void voronoiPolygons() {
     MultiPoint multiPoint = GEOMETRY_FACTORY.createMultiPointFromCoords(coordArray(0, 0, 2, 2));
     Geometry actual1 = FunctionsGeoTools.voronoiPolygons(multiPoint, 0, null);
-    assertEquals(
+    assertGeometryEquals(
         "GEOMETRYCOLLECTION (POLYGON ((-2 -2, -2 4, 4 -2, -2 -2)), POLYGON ((-2 4, 4 4, 4 -2, -2 4)))",
         actual1.toText());
 
     Geometry actual2 = FunctionsGeoTools.voronoiPolygons(multiPoint, 30, null);
-    assertEquals(
+    assertGeometryEquals(
         "GEOMETRYCOLLECTION (POLYGON ((-2 -2, -2 4, 4 4, 4 -2, -2 -2)))", actual2.toText());
 
     Geometry buf = Functions.buffer(GEOMETRY_FACTORY.createPoint(new Coordinate(1, 1)), 10);
     Geometry actual3 = FunctionsGeoTools.voronoiPolygons(multiPoint, 0, buf);
-    assertEquals(
+    assertGeometryEquals(
         "GEOMETRYCOLLECTION (POLYGON ((-9 -9, -9 11, 11 -9, -9 -9)), POLYGON ((-9 11, 11 11, 11 -9, -9 11)))",
         actual3.toText());
 
     Geometry actual4 = FunctionsGeoTools.voronoiPolygons(multiPoint, 30, buf);
-    assertEquals(
+    assertGeometryEquals(
         "GEOMETRYCOLLECTION (POLYGON ((-9 -9, -9 11, 11 11, 11 -9, -9 -9)))", actual4.toText());
 
     Geometry actual5 = FunctionsGeoTools.voronoiPolygons(null, 0, null);
@@ -3827,20 +3824,20 @@ public void maximumInscribedCircle() throws ParseException {
     InscribedCircle actual = Functions.maximumInscribedCircle(geom);
     InscribedCircle expected =
         new InscribedCircle(
-            Constructors.geomFromEWKT("POINT (96.953125 76.328125)"),
-            Constructors.geomFromEWKT("POINT (140 90)"),
-            45.165846);
-    assertTrue(expected.equals(actual));
+            Constructors.geomFromEWKT("POINT (96.9287109375 76.3232421875)"),
+            Constructors.geomFromEWKT("POINT (61.64205411585366 104.55256764481707)"),
+            45.18896951053177);
+    assertEquals(expected, actual);
 
     geom =
         Constructors.geomFromEWKT(
             "MULTILINESTRING ((59.5 17, 65 17), (60 16.5, 66 12), (65 12, 69 17))");
     actual = Functions.maximumInscribedCircle(geom);
     expected =
         new InscribedCircle(
-            Constructors.geomFromEWKT("POINT (65.0419921875 15.1005859375)"),
-            Constructors.geomFromEWKT("POINT (65 17)"),
-            1.8998781);
+            Constructors.geomFromEWKT("POINT (65.043212890625 15.098388671875)"),
+            Constructors.geomFromEWKT("POINT (66.52827267530488 13.910340844131097)"),
+            1.9018044602641058);
     assertTrue(expected.equals(actual));
 
     geom =
@@ -3849,9 +3846,9 @@ public void maximumInscribedCircle() throws ParseException {
     actual = Functions.maximumInscribedCircle(geom);
     expected =
         new InscribedCircle(
-            Constructors.geomFromEWKT("POINT (65.44062499999998 15.953124999999998)"),
-            Constructors.geomFromEWKT("POINT (67.5 16.9)"),
-            2.2666269);
+            Constructors.geomFromEWKT("POINT (65.44223632812499 15.945361328125001)"),
+            Constructors.geomFromEWKT("POINT (67.4 14.8)"),
+            2.2681911662992174);
     assertTrue(expected.equals(actual));
 
     geom = Constructors.geomFromEWKT("MULTIPOINT ((60.8 15.5), (63.2 16.3))");
@@ -3869,9 +3866,9 @@ public void maximumInscribedCircle() throws ParseException {
     actual = Functions.maximumInscribedCircle(geom);
     expected =
         new InscribedCircle(
-            Constructors.geomFromEWKT("POINT (65.44062499999998 15.953124999999998)"),
-            Constructors.geomFromEWKT("POINT (67.5 16.9)"),
-            2.2666269);
+            Constructors.geomFromEWKT("POINT (65.44223632812499 15.945361328125001)"),
+            Constructors.geomFromEWKT("POINT (67.4 14.8)"),
+            2.2681911662992174);
     assertTrue(expected.equals(actual));
   }
 
@@ -3994,6 +3991,44 @@ public void points() throws ParseException {
     assertEquals("MULTIPOINT Z((0 0 1), (1 1 2), (2 2 3), (0 0 1))", result1);
   }
 
+  @Test
+  public void scale() throws ParseException {
+    Geometry geom = Constructors.geomFromWKT("POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))", 0);
+    Geometry actual = Functions.scale(geom, 3, 2);
+    String expected = "POLYGON ((0 0, 0 2, 3 2, 3 0, 0 0))";
+    assertEquals(expected, actual.toString());
+
+    geom = Constructors.geomFromWKT("LINESTRING(0 1, 1 0)", 0);
+    actual = Functions.scale(geom, 10, 5);
+    expected = "LINESTRING (0 5, 10 0)";
+    assertEquals(expected, actual.toString());
+
+    geom = Constructors.geomFromWKT("POLYGON ((0 0, 0 1.5, 1.5 1.5, 1.5 0, 0 0))", 1111);
+    actual = Functions.scaleGeom(geom, Constructors.point(1.8, 2.1));
+    expected = "POLYGON ((0 0, 0 3.1500000000000004, 2.7 3.1500000000000004, 2.7 0, 0 0))";
+    assertEquals(expected, actual.toString());
+    assertEquals(1111, actual.getSRID());
+
+    actual =
+        Functions.scaleGeom(geom, Constructors.point(3, 2), Constructors.point(0.32959, 0.796483));
+    expected =
+        "POLYGON ((-0.6591799999999999 -0.796483, -0.6591799999999999 2.2035169999999997, 3.84082 2.2035169999999997, 3.84082 -0.796483, -0.6591799999999999 -0.796483))";
+    assertEquals(expected, actual.toString());
+
+    // test to check Z and M ordinate preservation
+    geom = Constructors.geomFromWKT("POLYGON ((0 0 1, 0 1.5 2, 1.5 1.5 2, 1.5 0 3, 0 0 1))", 0);
+    String actualWKT = Functions.asWKT(Functions.scale(geom, 3, 2));
+    expected = "POLYGON Z((0 0 1, 0 3 2, 4.5 3 2, 4.5 0 3, 0 0 1))";
+    assertEquals(expected, actualWKT);
+
+    geom =
+        Constructors.geomFromWKT(
+            "POLYGON ZM((0 0 1 2, 0 1.5 2 2, 1.5 1.5 2 2, 1.5 0 3 2, 0 0 1 2))", 0);
+    actualWKT = Functions.asWKT(Functions.scale(geom, 3, 2));
+    expected = "POLYGON ZM((0 0 1 2, 0 3 2 2, 4.5 3 2 2, 4.5 0 3 2, 0 0 1 2))";
+    assertEquals(expected, actualWKT);
+  }
+
   @Test
   public void rotateX() throws ParseException {
     Geometry lineString = Constructors.geomFromEWKT("LINESTRING (50 160, 50 50, 100 50)");